diff --git a/.git-blame-ignore-revs b/.git-blame-ignore-revs new file mode 100644 index 000000000..ac49978ce --- /dev/null +++ b/.git-blame-ignore-revs @@ -0,0 +1,6 @@ +# Revisions listed here are skipped by `git blame`. +# Enable locally with: +# git config blame.ignoreRevsFile .git-blame-ignore-revs +# +# (style) mechanical format sweep: isort then black over the Python tree (READ-133) +3fa941fa3e49fb647b94916b92d11e481e7b7fdd diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 7ebbf5cb1..6362e061a 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,40 +1,70 @@ name: Bug Report -description: Create a bug report to help us improve -title: "Bug Report" -labels: bug +description: Something in Condor does not work the way it should +title: "[Bug] " +labels: [bug] body: - - type: markdown - attributes: - value: | - ## **Before Submitting:** - - * Please edit the "Bug Report" to the title of the bug or issue - * Please make sure to look on our GitHub issues to avoid duplicate tickets - * You can add additional `Labels` to support this ticket (connectors, strategies, etc) - * If this is something to do with installation and how to's we would recommend to visit our [Hummingbot docs](https://hummingbot.org/docs/) and [Discord server](https://discord.gg/hummingbot) - - type: textarea - id: what-happened - attributes: - label: Describe the bug - description: A clear and concise description of the bug or issue. Please make sure to add screenshots and error message to help us investigate - placeholder: Tell us what happened? - validations: - required: true - - type: textarea - id: reproduce - attributes: - label: Steps to reproduce - description: A concise description of the steps to reproduce the buggy behavior - value: | - 1. - 2. - 3. - validations: - required: true - - type: textarea - id: attachment - attributes: - label: Attach required files - description: Please attach your config file and log file located on the "../gateway/logs/" folder. It would be really helpful to triage the issue. - validations: - required: false + - type: markdown + attributes: + value: | + Thanks for taking the time to report this. + + **Before you file:** search the [open issues](https://github.com/hummingbot/condor/issues?q=is%3Aissue) first — a duplicate splits the discussion in two. + + **Never paste secrets.** API keys, wallet private keys, seed phrases, your `.env` or an unredacted `config.yml` do not belong in a public issue. If the dashboard filled in the "Environment" box below, read it before you submit. + + - type: dropdown + id: area + attributes: + label: Area + description: Where in Condor did this happen? (The dashboard preselects this for you.) + options: + - Agents & chat + - Portfolio + - Bots & controllers + - Trading & executors + - Routines & reports + - Settings & connections + - Dashboard (other) + - Telegram bot + - Gateway / DEX + - Not sure + validations: + required: true + + - type: textarea + id: what-happened + attributes: + label: What happened + description: What you expected, and what you got instead. Screenshots help. + placeholder: The portfolio chart stays blank after I switch servers, but the balances table updates. + validations: + required: true + + - type: textarea + id: reproduce + attributes: + label: Steps to reproduce + description: The shortest path from a fresh session to the bug. + value: | + 1. + 2. + 3. + validations: + required: true + + - type: textarea + id: attachment + attributes: + label: Environment + description: | + Condor version, page, browser and recent errors. The dashboard's "Report an issue" button (🐛, top right) fills this in automatically; otherwise paste the output of `git rev-parse --short HEAD` and how you run Condor (source, Docker). + validations: + required: false + + - type: textarea + id: logs + attributes: + label: Logs + description: Relevant lines from the bot log or the browser console. Redact keys, hosts and wallet addresses first. + validations: + required: false diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index 13eb0bc9e..147822c1f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,35 +1,58 @@ -name: Feature request -description: Suggest an idea that will improve the Hummingbot codebase -title: "Feature Request" -labels: enhancement +name: Feature Request +description: Suggest something Condor should be able to do +title: "[Feature] " +labels: [enhancement] body: - - type: markdown - attributes: - value: | - ## **Before Submitting:** + - type: markdown + attributes: + value: | + Thanks for the idea. - * Please edit the "Feature Request" to the title of the feature - * Please make sure to look on our GitHub issues to avoid duplicate tickets - * You can add additional `Labels` to support this ticket (connectors, strategies, etc) - * If this is something to do with installation and how to's we would recommend to visit our [Discord server](https://discord.gg/hummingbot) and [Hummingbot docs](https://hummingbot.org/docs/) - - type: textarea - id: feature-suggestion - attributes: - label: Feature Suggestion - description: A clear and concise description of the feature request. If you have looked at the code and know exactly what code changes are needed then please consider submitting a pull request instead. - placeholder: How you want to achieve the desired behavior - validations: - required: true - - type: textarea - id: feature-impact - attributes: - label: Impact - description: A succinct description of why you want the desired behavior specified above. - placeholder: The desired behavior will allow me to.. - validations: - required: true - - type: textarea - id: feature-additional-context - attributes: - label: Additional context - description: Add any other context or screenshots about the feature request here (optional) + **Before you file:** search the [open issues](https://github.com/hummingbot/condor/issues?q=is%3Aissue) — someone may already be arguing for it, and a second voice on one thread carries further than a second thread. + + If you already know what the change looks like in code, a pull request is welcome instead. + + - type: dropdown + id: area + attributes: + label: Area + description: Which part of Condor would this touch? (The dashboard preselects this for you.) + options: + - Agents & chat + - Portfolio + - Bots & controllers + - Trading & executors + - Routines & reports + - Settings & connections + - Dashboard (other) + - Telegram bot + - Gateway / DEX + - Not sure + validations: + required: true + + - type: textarea + id: feature-suggestion + attributes: + label: What should it do + description: Describe the behavior you want, not the implementation. + placeholder: Let me filter the executors table by connector, the way the bots table already filters by status. + validations: + required: true + + - type: textarea + id: feature-impact + attributes: + label: Why it matters + description: What this would let you do that you cannot do today, and how often you hit it. + placeholder: With 40 executors open I scroll past everything to find the two on Hyperliquid. + validations: + required: true + + - type: textarea + id: feature-additional-context + attributes: + label: Additional context + description: Mockups, links, the workaround you use today, or the version you are on. + validations: + required: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 000000000..e7678cf7c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,103 @@ +name: CI + +on: + push: + branches: [main] + pull_request: + branches: [main] + +jobs: + backend-lint: + name: Backend Lint + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: uv sync --dev + + - name: Check formatting with black + run: uv run black --check . + + - name: Check import sorting with isort + run: uv run isort --check . + + backend-import-check: + name: Backend Import Check + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: uv sync + + - name: Verify main modules import without errors + run: | + uv run python -c " + import config_manager + import handlers + import utils + print('All core modules imported successfully') + " + + backend-tests: + name: Backend Tests + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + + - uses: astral-sh/setup-uv@v4 + with: + version: "latest" + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + + - name: Install dependencies + run: uv sync --dev + + - name: Run tests + run: uv run pytest --tb=short -q + + frontend-build: + name: Frontend Build & Lint + runs-on: ubuntu-latest + defaults: + run: + working-directory: frontend + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-node@v4 + with: + node-version: "20" + cache: "npm" + cache-dependency-path: frontend/package-lock.json + + - name: Install dependencies + run: npm ci + + # Gates on *new* errors against eslint-baseline.json rather than on the + # 76 that predate this workflow -- see frontend/scripts/lint-baseline.mjs. + - name: Lint + run: npm run lint:ci + + - name: Build + run: npm run build diff --git a/PRIVACY.md b/PRIVACY.md new file mode 100644 index 000000000..3cfc426f3 --- /dev/null +++ b/PRIVACY.md @@ -0,0 +1,143 @@ +# Privacy + +Condor is self-hosted. It runs on your machine, holds your exchange API keys, +and places your orders. So the default for anything that could leave that +machine is **off**, and this document is the complete statement of what the one +optional exception does. + +**Short version:** a fresh install sends nothing. It has no telemetry consent +recorded, which resolves to level `off`, at which the emitter returns before it +has looked at its arguments — no buffer, no file, no directory. Nothing changes +until an admin taps "yes" on a prompt. Even then, this repository has no +collector address compiled into it: with `CONDOR_TELEMETRY_URL` unset, the send +path is inert and events can only ever reach a capped local file. + +The whole mechanism is about 900 lines in [`condor/telemetry/`](condor/telemetry/). +It is meant to be read, not trusted. + +--- + +## What is collected, if you opt in + +There are three levels. You choose one; you can change it later. + +| Level | What it sends | +|---|---| +| `off` | Nothing, ever. The emitter is a no-op. **This is the default.** | +| `ping` | Only that this install exists: `install`, `heartbeat`, `version_change`, `shutdown`. | +| `usage` | The above plus the feature, reliability and agent events below. | + +Every batch carries one context block describing the *deployment*, not you: + +| Field | Example | Why | +|---|---|---| +| `install_id` | a random UUID | Count installs and retention. Generated once, from `uuid4`. Not derived from your MAC, hostname, username, or any token. | +| `app.version`, `app.branch` | `54ad4dc`, `main` | Which commit is actually running, so we know what to support. | +| `app.python`, `app.os`, `app.arch`, `app.in_docker` | `3.12`, `linux`, `arm64`, `true` | What to test against. | +| `config.*` | `has_gateway: true`, `user_count: 3`, `server_count: 2`, `llm_providers: ["openai"]` | Counts and capability flags. Numbers and fixed provider *names* — never a server name, never a URL, never a key. | +| `dropped` | `0` | How many events the rate limiter discarded. An honest count, so a quiet incident is visibly quiet. | + +And the events themselves: + +| Event | What it carries | +|---|---| +| `command` | Which of our ~20 commands was used (anything else is `other`), the surface, and whether the sender was an approved user. | +| `action` | A module (`bots`, `dex`, `agents`, …) and a verb (`view`, `deploy`, `start`). On the web, derived from the matched route *template* — `/api/v1/bots/{name}`, never the URL. | +| `feature_first_use` | The first time this install ever uses a feature. | +| `bot_deploy`, `executor_deploy`, `trade` | Connector name, controller/executor type, side, order type, paper-or-not. | +| `routine_run` | The routine's name **only if it ships in this repo** — a routine you wrote is reported as `custom` — plus how it was triggered, how long it took, and whether it worked. | +| `error` | The exception *type*, a SHA-256 hash of its message, and up to five `file:line` frames from our own packages, relative to the repo root. | +| `upstream_error` | Which dependency (`hb_api`, `gateway`, `llm`, `telegram`), which operation, which status code. | +| `agent_turn` | Provider and model id, tool-call count by category, duration, outcome. | +| `mcp_tool` | Which MCP tool ran, whether it worked, how long it took. | +| `strategy_run` | Execution mode, tick frequency, tick count, whether risk limits were configured, why it stopped. | +| `confirmation` | Which tool was asked about, and whether it was allowed, denied, or timed out. | +| `heartbeat` | Uptime, a *count* of distinct users active in the last 24h, and a per-surface activity count. | + +## What is never collected + +This is not a policy the call sites are asked to respect. Every event and every +property is declared in [`condor/telemetry/schema.py`](condor/telemetry/schema.py), +and anything undeclared is **dropped** on the way in. Free-form strings are +truncated to 64 characters and stripped to identifier-shaped characters, so +nothing long enough to be a key, an address, a prompt or a URL survives intact. +[`tests/test_telemetry.py`](tests/test_telemetry.py) asserts it. + +Never sent, under any level: + +- **Secrets** — API keys, secret keys, passphrases, private keys, your Telegram bot token. +- **Money** — order amounts, balances, portfolio value, PnL, position sizes, leverage. +- **Positions** — trading pairs. Deliberately excluded: an install that trades one pair, plus event timestamps, is a deanonymizable disclosure of what you hold. Connector *names* are sent; pairs are not. +- **Addresses and identifiers** — wallet addresses, order ids, transaction hashes. +- **Your infrastructure** — server names, URLs, hostnames, IP addresses, file paths outside this repo, your home directory or username. +- **People** — Telegram user ids, usernames, chat ids, display names. +- **Content** — prompts, agent replies, journal entries, notes, routine configs, report bodies, and exception *message* strings. + +Two of those deserve a note, because they are where this kind of thing usually +leaks: + +- **Exception messages are hashed, not sent.** A message string is the single + most common carrier of a balance, a hostname or a key. We send + `sha256(message)[:12]`, which groups identical errors together and discloses + nothing. +- **Agent tool calls are counted by category, not by title.** A tool call's + title is free text and routinely contains a file path. We count the ACP + `kind` (`read`, `execute`, …) instead. + +### The one derived identifier + +`user_hash` lets the collector count how many distinct people use an install +without knowing who they are. It is `sha256(install_secret + telegram_user_id)`, +truncated to 16 characters, where `install_secret` is a random UUID that **never +leaves your machine**. The Telegram id is never sent, and because the salt is +per-install, the same person on two installs produces two unrelated hashes. + +## Where it goes + +Nowhere, unless you configure a destination. + +No collector URL is compiled into this repository. Events are buffered in memory +and, when a batch cannot be delivered, appended to +`condor/.runtime/telemetry/outbox.jsonl`, which is capped at 5,000 events and 7 +days — oldest dropped. With `CONDOR_TELEMETRY_URL` unset, that is the entire +life cycle: a local file, capped, that you can delete. + +You can read exactly what would be sent: + +```bash +cat condor/.runtime/telemetry/outbox.jsonl | jq . +``` + +## How to turn it off — or check it is off + +It is already off unless you turned it on. To be certain: + +```bash +# The authoritative answer. Prints "off" on a default install. +uv run python -c "from condor.telemetry import consent; print(consent.level())" +``` + +Three ways to control it, in order of precedence: + +1. **Environment** — `CONDOR_TELEMETRY=off` in your `.env`. This overrides + everything, in both directions, and is the right answer for a headless or + containerized install. `CONDOR_TELEMETRY_URL` is what enables sending at all; + leave it unset and nothing can be transmitted. +2. **The dashboard API** — `PUT /api/v1/settings/telemetry?level=off` (admin + only). `GET` the same path to see the current state. +3. **`config.yml`** — edit the `telemetry` section directly: + + ```yaml + telemetry: + consent: denied + level: off + ``` + +Turning it off is a **withdrawal, not a pause**: the in-memory buffer and the +outbox file are deleted, so nothing already recorded can be sent afterwards. + +## Changes to this document + +Adding anything to the collected list requires a change to `schema.py`, a change +to this file, and re-asking for consent. In particular, adding trading pairs +would make positions inferable from timing and must not be done quietly. diff --git a/README.md b/README.md index 93a1795b2..fac5a7711 100644 --- a/README.md +++ b/README.md @@ -12,14 +12,23 @@ A Telegram bot for monitoring and trading with Hummingbot via the **Hummingbot A > > Full walkthrough: [Securing Condor and Hummingbot API with Tailscale](https://hummingbot.org/blog/posts/securing-condor-and-hummingbot-api-with-tailscale/) · [Hummingbot API Tailscale guide](https://hummingbot.org/hummingbot-api/tailscale/) +> **Privacy:** Condor collects nothing by default. It ships with usage telemetry +> **off** and no collector address, and it stays that way unless an admin +> explicitly opts in from the one prompt it sends on first boot. What that +> option would and would not collect — and how to verify it is off — is spelled +> out in [PRIVACY.md](PRIVACY.md). + ## Features - **Portfolio Dashboard** - Comprehensive portfolio view with PNL tracking, 24h changes, and graphical analysis - **Bot Monitoring** - Track active Hummingbot trading bots with real-time status and metrics - **CLOB Trading** - Place orders on centralized exchanges (Binance, Bybit, etc.) with interactive menus - **DEX Trading** - Swap tokens and manage CLMM liquidity positions via Gateway +- **Web Dashboard** - Browser UI (via **`/web`**) with a live trade panel that covers both CEX and DEX venues: order book trading, DEX swaps, and CLMM liquidity positions created and drawn directly on the chart - **Configuration** - Manage API servers, exchange credentials, and Gateway through Telegram (`/servers`, `/keys`, `/gateway`) - **AI Assistant** - Natural language trading help via **`/agent`** (optional OpenAI or OpenRouter keys, or any custom OpenAI-compatible endpoint like Venice AI; MCP tools when configured) +- **AI Agents** - Domain agents that can be consulted, delegated to, or run on a loop — each can author its own tick strategy (analysis order, decision rules, risk limits) and run it autonomously with dry-run support +- **Issue Reporting** - A **Report an issue** button in the web dashboard prefills a GitHub issue draft (with opt-in, review-before-send diagnostics) that you submit from your own GitHub account ## What you need @@ -409,8 +418,7 @@ See `flows/` directory for detailed command flow documentation: - **Docs**: https://condor.hummingbot.org - **Installation guide**: https://condor.hummingbot.org/getting-started/installing - **Tailscale guide**: https://hummingbot.org/hummingbot-api/tailscale/ -- **Discord**: [#condor-feedback](https://discord.gg/hummingbot) -- **Issues**: https://github.com/hummingbot/condor/issues +- **Issues**: https://github.com/hummingbot/condor/issues — or use the **Report an issue** button in the web dashboard to prefill a bug report or feature request with diagnostics --- diff --git a/agents/_shared/skills/routine_cookbook/SKILL.md b/agents/_shared/skills/routine_cookbook/SKILL.md index 0cf7d335c..bc2b53bb9 100644 --- a/agents/_shared/skills/routine_cookbook/SKILL.md +++ b/agents/_shared/skills/routine_cookbook/SKILL.md @@ -29,6 +29,24 @@ Most routines need `report_builder.md` plus one or two others. A continuous price monitor with a live dashboard, for example, reads `hummingbot_client.md` + `continuous.md`. +## First: is this a routine at all? + +A routine is a **durable artifact** — it has a name, a config schema, a place in +the library, and it can be scheduled, shared and re-run by anyone. That is worth +a file when the work repeats. + +A **one-off computation** is not. "What were SOL's hourly returns yesterday", +"what is the spread between these two venues right now", "aggregate these +executors by controller" — write the Python and call +`run_code(code="...")`. It runs in the bot with exactly the primitives below +(`context`, `client`, pandas, `ReportBuilder`, every `condor.*` module), returns +its `print` output and its `result`, and hands you the traceback to fix when it +fails. No file, no `Config` class, no library entry. + +Promote a snippet to a routine when you have run essentially the same thing a +third time, or the moment it needs to be scheduled, shared, or visible to the +user. + ## Where the routine lives **Agent-local** — `agents/{slug}/routines/` — visible only to that agent, and diff --git a/agents/_shared/skills/strategy_builder/SKILL.md b/agents/_shared/skills/strategy_builder/SKILL.md new file mode 100644 index 000000000..dacdd7758 --- /dev/null +++ b/agents/_shared/skills/strategy_builder/SKILL.md @@ -0,0 +1,137 @@ +--- +name: strategy_builder +description: Give YOURSELF a loop — write a tick playbook (a strategy) under your own slug, dry-run it, and start it. The shared contract for how a strategy is authored, validated and launched; read by any agent that wants to act autonomously instead of only being consulted. +when_to_use: You are an agent and the user wants you to act on a loop — run every N seconds, watch a condition, report on a schedule, or trade autonomously. Also read this before editing or re-launching a strategy you already own. NOT for creating or deleting other agents (that is Condor's agent_builder). +created: '2026-08-11T00:00:00Z' +source: chat +--- + +# Strategy Builder + +You are an **agent**. You can already be consulted, delegated to, and looped — from the +moment you existed. This playbook covers the last of those: giving your loop a **specific +tick playbook** instead of the generic default. + +A **strategy** is a tick system prompt the engine runs in a **session**, at a frequency +the user sets. It lives at `agents/{your_slug}/strategies/{strategy_slug}/strategy.md` and +it is yours — you author it, under your own slug. + +> **Scope.** This is about giving *yourself* a loop. Creating, editing or deleting *other* +> agents belongs to Condor (`agent_builder`). If the user wants a whole new specialist, +> say so and let them ask Condor. + +## When a strategy is worth it + +`start_agent(strategy_id="")` already works with no strategy — it ticks a +default playbook driven by your AGENT.md. That default is deliberately generic. Write a +strategy when the loop needs to be **specific and disciplined**: a fixed analysis order, a +decision rule, executor schemas, risk limits. + +**The loop does NOT have to trade.** Define the tick task however the user wants: + +- read a routine's output and **decide whether to trade** (create/stop executors), +- or just **send a report / notification**, +- or watch a condition and act only when it's met. + +## Step 1 — Prepare (only if it trades) + +BEFORE writing the strategy, fetch the schema for every executor type the loop will use — +`manage_executors(executor_type="grid_strike")`, etc. — and embed the required +fields/types directly into the instructions. The tick LLM has no other way to learn them. +Same for any controller config it manages (`manage_controllers`). + +If the loop should reason over structured data, make sure the routine exists first +(`manage_routines(action="list", agent="")`). Routines are agent-scoped, so any +strategy you own can call them. Need a new one? Do NOT write it inline — hand it to a +background worker: `delegate(action="start", agent="condor", task="build a routine that … +for agent ")`. + +## Step 2 — Create the strategy + +``` +manage_trading_agent( + action="create_strategy", + agent_slug="", # yourself + name="BRL MM", + description="…", + instructions="", + # agent_key omitted → inherits your model; overridable at launch + config={"connector_name": "binance", "frequency_sec": 60, + "total_amount_quote": 100, "execution_mode": "loop"} +) +``` + +Returns the composite key `"."` — that is the `strategy_id` for +everything after. + +The tick instructions MUST include: + +- **Objective** — what one tick is for. +- **Analysis** — which routine to call *by name* and how to read its output. +- **Decision logic** — act / report / hold, with the condition for each. + +…and, only if it trades: + +- **Executor config** — the FULL schema: every required field, type, range, ordering rule. +- **Parameter inference** — how to derive prices/side/TP from routine output + market data. +- **Risk rules** — max position, position limits, stop behaviour. +- **Error recovery** — on a failed create, re-fetch the schema, fix, retry once, journal it. + +**Generic vs specific:** +- GENERIC (default): pair/connector are NOT in the instructions — they arrive at launch via + `trading_context`. Refer to "the configured trading pair"; keep a sensible `default_config`. +- SPECIFIC: pair/connector baked in (e.g. an ETH/BTC ratio play). + +## Step 3 — Dry run before live (if it trades) + +``` +manage_trading_agent(action="start_agent", strategy_id="", + config={"execution_mode": "dry_run", + "trading_context": "Trade BTC-USDT on binance_perpetual", + "frequency_sec": 60, "total_amount_quote": 100, + "risk_limits": {"max_position_size_quote": 200, "max_open_executors": 3}}) +``` + +Review with `trading_agent_journal_read(agent_id=…, section="run:1")`: routines called +right, decision logic sound, conditional language ("would place…"), no real create/stop +calls, risk rules respected. Do not go live until the user is satisfied. + +> A `dry_run` / `run_once` is an **experiment** — it writes a snapshot, not a journal +> session. Read it back the same way; don't expect a numbered run to persist. + +## Step 4 — Go live + +Offer `run_once` (single live tick), `loop` (continuous), or `loop` + `max_ticks`. Confirm +the model, start it, confirm it's running, and give the user the monitoring commands. +**Always include risk limits when the loop can trade.** + +## Monitoring what you own + +1. `manage_trading_agent(action="list_agent_definitions")` — agents and the strategies they own. +2. `manage_trading_agent(action="list_agents")` — running loop instances. +3. `manage_trading_agent(action="agent_status", agent_id=…)` — instance status. +4. `trading_agent_journal_read(agent_id=…, section="summary"|"runs"|"run:N")`. + +## Reference + +**Model:** the strategy's `agent_key` defaults to yours; override per launch with +`config={"agent_key": "…"}`. Never invent one — call `get_available_models` and pick from +what the operator actually has, or leave it inherited. A pydantic-ai key +(`ollama:`/`openai:`/`groq:`/`lmstudio:`/`openrouter:`/`custom@…`) enforces the `tools` +allowlist; an ACP key (`claude-code`/`claude-acp`/`gemini`/`copilot`) runs unrestricted, +with mutations still confirmation-gated. + +**Server:** leave `server_name` empty unless the user pins it. A strategy you own runs on +whichever server your agent resolves. + +**Editing & deleting:** `update_strategy(strategy_id=…, instructions=…)` to revise the tick +playbook; `delete_strategy(strategy_id=…)` to remove it. Stop any running instance first. + +## Rules + +- Author strategies **under your own slug only**. Another agent's fleet is not yours to edit. +- Fetch executor/controller schemas BEFORE writing instructions that use them. +- One strategy per loop job — don't overload a tick with unrelated objectives. +- A loop doesn't have to trade. When it does: risk limits always, dry run always. +- Show the user the dry-run journal before proposing to go live. +- Don't write routine code inline — delegate it to a Condor worker. diff --git a/agents/adaptive_grid_trader/routines/baseline_7d.py b/agents/adaptive_grid_trader/routines/baseline_7d.py index c63167127..2454b8f13 100644 --- a/agents/adaptive_grid_trader/routines/baseline_7d.py +++ b/agents/adaptive_grid_trader/routines/baseline_7d.py @@ -1,7 +1,9 @@ +import logging + from pydantic import BaseModel, Field from telegram.ext import ContextTypes + from config_manager import get_client -import logging logger = logging.getLogger(__name__) diff --git a/agents/adaptive_grid_trader/routines/hourly_mtf_check.py b/agents/adaptive_grid_trader/routines/hourly_mtf_check.py index 7d72c43c2..34693610f 100644 --- a/agents/adaptive_grid_trader/routines/hourly_mtf_check.py +++ b/agents/adaptive_grid_trader/routines/hourly_mtf_check.py @@ -1,10 +1,12 @@ -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes -from config_manager import get_client import asyncio import logging import math +from pydantic import BaseModel, Field +from telegram.ext import ContextTypes + +from config_manager import get_client + logger = logging.getLogger(__name__) CATEGORY = "Analysis" @@ -12,17 +14,25 @@ class Config(BaseModel): """Multi-timeframe analysis → LONG_GRID / SHORT_GRID / TWO_SIDED_GRID / HOLD recommendation.""" + trading_pair: str = Field(default="BTC-USDT") connector_name: str = Field(default="binance_perpetual") - lifetime_hours: float = Field(default=8.0, description="Expected grid lifetime for range calculation (6-12h typical due to anti-flip rule)") + lifetime_hours: float = Field( + default=8.0, + description="Expected grid lifetime for range calculation (6-12h typical due to anti-flip rule)", + ) atr_period: int = Field(default=14) - baseline_atr: float = Field(default=0.0, description="From baseline_7d — 0 means compute from 1h candles only") + baseline_atr: float = Field( + default=0.0, + description="From baseline_7d — 0 means compute from 1h candles only", + ) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- + def _parse_candles(result) -> list: """Defensively parse candles from API response.""" if result is None: @@ -115,7 +125,9 @@ def _confidence(trend_4h: str, trend_1d: str, closes_1h: list) -> str: slow = _compute_ema(closes_1h, 21) if fast and slow and slow[-1]: diff = (fast[-1] - slow[-1]) / slow[-1] * 100 - aligned = (trend_4h == "BULLISH" and diff > 0.3) or (trend_4h == "BEARISH" and diff < -0.3) + aligned = (trend_4h == "BULLISH" and diff > 0.3) or ( + trend_4h == "BEARISH" and diff < -0.3 + ) return "HIGH" if aligned else "MEDIUM" return "MEDIUM" @@ -124,6 +136,7 @@ def _confidence(trend_4h: str, trend_1d: str, closes_1h: list) -> str: # Main # --------------------------------------------------------------------------- + async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: client = await get_client(context._chat_id, context=context) if not client: @@ -132,11 +145,17 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: # -- 1. Fetch three timeframes in parallel -- try: raw_1h, raw_4h, raw_1d = await asyncio.gather( - client.market_data.get_candles(config.connector_name, config.trading_pair, "1h", max_records=50), + client.market_data.get_candles( + config.connector_name, config.trading_pair, "1h", max_records=50 + ), # _trend_direction needs slow EMA period + 2 = 23 candles minimum, # so keep a margin above that or the timeframe reads NEUTRAL forever. - client.market_data.get_candles(config.connector_name, config.trading_pair, "4h", max_records=60), - client.market_data.get_candles(config.connector_name, config.trading_pair, "1d", max_records=60), + client.market_data.get_candles( + config.connector_name, config.trading_pair, "4h", max_records=60 + ), + client.market_data.get_candles( + config.connector_name, config.trading_pair, "1d", max_records=60 + ), ) except Exception as e: logger.error(f"[hourly_mtf_check] candle fetch failed: {e}") @@ -156,7 +175,9 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: atr_1h = (atr_1h + config.baseline_atr) / 2.0 window_24 = candles_1h[-24:] if len(candles_1h) >= 24 else candles_1h - vol_level, range_high, range_low, range_pct, range_pos = _volatility_level(atr_1h, window_24) + vol_level, range_high, range_low, range_pct, range_pos = _volatility_level( + atr_1h, window_24 + ) current_price = float(candles_1h[-1].get("close", 0) or 0) # 4h: trend direction @@ -168,20 +189,28 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: # -- 3. Signal synthesis -- if trend_4h == "BULLISH" and trend_1d == "BULLISH": profile = "LONG_GRID" - rationale = ("Both medium (4h) and macro (1d) timeframes confirm bullish momentum. " - "Grid skewed upward — capture breakout while limiting downside exposure.") + rationale = ( + "Both medium (4h) and macro (1d) timeframes confirm bullish momentum. " + "Grid skewed upward — capture breakout while limiting downside exposure." + ) elif trend_4h == "BEARISH" and trend_1d == "BEARISH": profile = "SHORT_GRID" - rationale = ("Both medium (4h) and macro (1d) timeframes confirm bearish pressure. " - "Grid skewed downward — profit from continued decline with capped upside risk.") + rationale = ( + "Both medium (4h) and macro (1d) timeframes confirm bearish pressure. " + "Grid skewed downward — profit from continued decline with capped upside risk." + ) elif vol_level in ("LOW", "MODERATE"): profile = "TWO_SIDED_GRID" - rationale = ("Timeframes disagree or both neutral with contained volatility — " - "price is likely range-bound. Symmetric two-sided grid maximises fee capture.") + rationale = ( + "Timeframes disagree or both neutral with contained volatility — " + "price is likely range-bound. Symmetric two-sided grid maximises fee capture." + ) else: profile = "HOLD" - rationale = ("Conflicting signals combined with elevated volatility — " - "directional conviction is insufficient. Avoid new grid entry.") + rationale = ( + "Conflicting signals combined with elevated volatility — " + "directional conviction is insufficient. Avoid new grid entry." + ) closes_1h = [float(c.get("close", 0) or 0) for c in candles_1h] conf = _confidence(trend_4h, trend_1d, closes_1h) @@ -209,29 +238,85 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: # -- Build levels table -- if profile == "LONG_GRID": levels_rows = [ - {"Level": "limit_price", "Price": f"{limit_price:,.4f}", "Description": "Abort / stop-loss if breached"}, - {"Level": "start_price", "Price": f"{start_price:,.4f}", "Description": "Grid lower bound"}, - {"Level": "current_price", "Price": f"{current_price:,.4f}", "Description": "Reference at analysis time"}, - {"Level": "end_price", "Price": f"{end_price:,.4f}", "Description": "Grid upper bound / TP zone"}, + { + "Level": "limit_price", + "Price": f"{limit_price:,.4f}", + "Description": "Abort / stop-loss if breached", + }, + { + "Level": "start_price", + "Price": f"{start_price:,.4f}", + "Description": "Grid lower bound", + }, + { + "Level": "current_price", + "Price": f"{current_price:,.4f}", + "Description": "Reference at analysis time", + }, + { + "Level": "end_price", + "Price": f"{end_price:,.4f}", + "Description": "Grid upper bound / TP zone", + }, ] elif profile == "SHORT_GRID": levels_rows = [ - {"Level": "end_price", "Price": f"{end_price:,.4f}", "Description": "Grid upper bound"}, - {"Level": "current_price", "Price": f"{current_price:,.4f}", "Description": "Reference at analysis time"}, - {"Level": "start_price", "Price": f"{start_price:,.4f}", "Description": "Grid lower bound / TP zone"}, - {"Level": "limit_price", "Price": f"{limit_price:,.4f}", "Description": "Abort / stop-loss if breached"}, + { + "Level": "end_price", + "Price": f"{end_price:,.4f}", + "Description": "Grid upper bound", + }, + { + "Level": "current_price", + "Price": f"{current_price:,.4f}", + "Description": "Reference at analysis time", + }, + { + "Level": "start_price", + "Price": f"{start_price:,.4f}", + "Description": "Grid lower bound / TP zone", + }, + { + "Level": "limit_price", + "Price": f"{limit_price:,.4f}", + "Description": "Abort / stop-loss if breached", + }, ] elif profile == "TWO_SIDED_GRID": levels_rows = [ - {"Level": "start_price", "Price": f"{start_price:,.4f}", "Description": "Grid lower bound (−2D)"}, - {"Level": "current_price", "Price": f"{current_price:,.4f}", "Description": "Reference / centre"}, - {"Level": "end_price", "Price": f"{end_price:,.4f}", "Description": "Grid upper bound (+2D)"}, + { + "Level": "start_price", + "Price": f"{start_price:,.4f}", + "Description": "Grid lower bound (−2D)", + }, + { + "Level": "current_price", + "Price": f"{current_price:,.4f}", + "Description": "Reference / centre", + }, + { + "Level": "end_price", + "Price": f"{end_price:,.4f}", + "Description": "Grid upper bound (+2D)", + }, ] else: levels_rows = [ - {"Level": "ref_low", "Price": f"{start_price:,.4f}", "Description": "Reference lower (−D) — NOT ACTIONABLE"}, - {"Level": "current_price", "Price": f"{current_price:,.4f}", "Description": "Reference at analysis time"}, - {"Level": "ref_high", "Price": f"{end_price:,.4f}", "Description": "Reference upper (+D) — NOT ACTIONABLE"}, + { + "Level": "ref_low", + "Price": f"{start_price:,.4f}", + "Description": "Reference lower (−D) — NOT ACTIONABLE", + }, + { + "Level": "current_price", + "Price": f"{current_price:,.4f}", + "Description": "Reference at analysis time", + }, + { + "Level": "ref_high", + "Price": f"{end_price:,.4f}", + "Description": "Reference upper (+D) — NOT ACTIONABLE", + }, ] # -- 5. ReportBuilder -- @@ -239,10 +324,15 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: from condor.reports import ReportBuilder builder = ReportBuilder(f"MTF Check — {config.trading_pair}") - builder.source("routine", "hourly_mtf_check").tags(["grid", "mtf", "analysis", "adaptive"]) + builder.source("routine", "hourly_mtf_check").tags( + ["grid", "mtf", "analysis", "adaptive"] + ) # Section 1: Timeframe Summary - builder.section("01 / TIMEFRAME SUMMARY", "Per-timeframe: trend direction, ATR/volatility, signal") + builder.section( + "01 / TIMEFRAME SUMMARY", + "Per-timeframe: trend direction, ATR/volatility, signal", + ) tf_rows = [ { "Timeframe": "1h (Range & Vol)", @@ -275,10 +365,25 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: "Signal": trend_1d, }, ] - builder.table(tf_rows, ["Timeframe", "Candles", "ATR(14)", "24h Range", "Range %", "Volatility", "Price in Range", "Signal"]) + builder.table( + tf_rows, + [ + "Timeframe", + "Candles", + "ATR(14)", + "24h Range", + "Range %", + "Volatility", + "Price in Range", + "Signal", + ], + ) # Section 2: Signal Synthesis - builder.section("02 / SIGNAL SYNTHESIS", "Agreement analysis across timeframes and final profile decision") + builder.section( + "02 / SIGNAL SYNTHESIS", + "Agreement analysis across timeframes and final profile decision", + ) builder.kpi("4h Trend", trend_4h) builder.kpi("1d Trend", trend_1d) builder.kpi("1h Volatility", vol_level) @@ -288,12 +393,18 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: f"**4h vs 1d agreement:** {agreement_str} \n" f"**Current price:** {current_price:,.4f} \n" f"**ATR(1h, {config.atr_period}):** {atr_1h:.6f}" - + (f" *(blended with baseline {config.baseline_atr})*" if config.baseline_atr > 0 else "") + - f" \n**D = ATR × √{config.lifetime_hours:.1f}h = {D:.6f}**" + + ( + f" *(blended with baseline {config.baseline_atr})*" + if config.baseline_atr > 0 + else "" + ) + + f" \n**D = ATR × √{config.lifetime_hours:.1f}h = {D:.6f}**" ) # Section 3: Recommendation - builder.section("03 / RECOMMENDATION", "Actionable grid profile with concrete price levels") + builder.section( + "03 / RECOMMENDATION", "Actionable grid profile with concrete price levels" + ) builder.kpi("Profile", profile) builder.kpi("Confidence", conf) builder.kpi("D Value", f"{D:.4f}") @@ -302,7 +413,9 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: actionable = profile != "HOLD" if limit_price is not None: builder.kpi("Limit Price", f"{limit_price:,.4f}") - builder.kpi("Start Price", f"{start_price:,.4f}" + ("" if actionable else " (ref)")) + builder.kpi( + "Start Price", f"{start_price:,.4f}" + ("" if actionable else " (ref)") + ) builder.kpi("End Price", f"{end_price:,.4f}" + ("" if actionable else " (ref)")) builder.table(levels_rows, ["Level", "Price", "Description"]) @@ -322,12 +435,18 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: f"Current price: {current_price:,.4f} | ATR(1h): {atr_1h:.6f} | D: {D:.6f}", ] if profile == "LONG_GRID": - lines.append(f"limit_price={limit_price:.4f} | start_price={start_price:.4f} | end_price={end_price:.4f}") + lines.append( + f"limit_price={limit_price:.4f} | start_price={start_price:.4f} | end_price={end_price:.4f}" + ) elif profile == "SHORT_GRID": - lines.append(f"start_price={start_price:.4f} | end_price={end_price:.4f} | limit_price={limit_price:.4f}") + lines.append( + f"start_price={start_price:.4f} | end_price={end_price:.4f} | limit_price={limit_price:.4f}" + ) elif profile == "TWO_SIDED_GRID": lines.append(f"start_price={start_price:.4f} | end_price={end_price:.4f}") else: - lines.append(f"ref_low={start_price:.4f} | ref_high={end_price:.4f} [NOT ACTIONABLE]") + lines.append( + f"ref_low={start_price:.4f} | ref_high={end_price:.4f} [NOT ACTIONABLE]" + ) lines.append(f"Rationale: {rationale}") return "\n".join(lines) diff --git a/agents/adaptive_grid_trader/routines/position_mode_check.py b/agents/adaptive_grid_trader/routines/position_mode_check.py index ed0ff2027..f2f405e71 100644 --- a/agents/adaptive_grid_trader/routines/position_mode_check.py +++ b/agents/adaptive_grid_trader/routines/position_mode_check.py @@ -1,7 +1,9 @@ +import logging + from pydantic import BaseModel, Field from telegram.ext import ContextTypes + from config_manager import get_client -import logging logger = logging.getLogger(__name__) @@ -119,9 +121,13 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: "so level math and the liquidation guard would both be wrong" ) if flat: - reason += ". Account is flat, so HEDGE could be set if two-sided is wanted" + reason += ( + ". Account is flat, so HEDGE could be set if two-sided is wanted" + ) else: - reason += ". Account is not flat, so the mode cannot be changed right now" + reason += ( + ". Account is not flat, so the mode cannot be changed right now" + ) lines = [f"Position Mode — {config.connector_name}", f"mode: {mode}"] if shrug_note: diff --git a/agents/backpack_mm/AGENT.md b/agents/backpack_mm/AGENT.md new file mode 100644 index 000000000..79987b40f --- /dev/null +++ b/agents/backpack_mm/AGENT.md @@ -0,0 +1,74 @@ +--- +name: Backpack MM +description: Backpack Market Maker Program specialist — volume analysis, market selection, + bot deployment, and ongoing MM operations on Backpack perps +agent_key: claude-acp:sonnet +tools: [] +when_to_consult: When the user wants to analyze Backpack exchange markets, deploy + or manage MM bots on Backpack perps, understand volume requirements for the MM program, + check bot performance against program thresholds, or rotate markets for the Backpack + MM Rewards Program +server_required: true +server_name: local +created_by: 481175164 +created_at: '2026-07-30T06:45:22.685798+00:00' +--- + +# Backpack MM Agent + +You are a specialist in the Backpack Exchange Market Maker Program. You analyze the volume landscape, select markets, deploy and manage pmm_mister bots, and track program qualification metrics. + +## What you do +- Analyze 24h volume per market on Backpack (spot + perpetuals) +- Compute how much maker volume is needed to reach program thresholds +- Assess spread / uptime requirements per market +- Recommend which markets are most tractable given capital constraints +- Deploy and manage pmm_mister bots on Backpack perps +- Monitor bot performance and adjust configs for program compliance +- Rotate markets when volume shifts make qualification harder/easier + +## Program knowledge — Backpack Market Maker Program + +**Reward pool:** $300,000/month — $200k perps, $100k spot + +**Tier score** = max(Total Maker Share, Adjusted Maker Share) +- Total Maker Share = user maker vol / total maker vol (no multipliers) +- Adjusted Maker Share = user adjusted vol / total adjusted vol (lower-liquidity markets get multipliers) + +**Blended Score** (determines reward share per market): +- (Volume Score × 0.8) + (Liquidity Score × 0.2) + +**Volume Score** = user maker vol / total maker vol on that market + +**Liquidity Score** = (Avg Order Size / Avg Spread) × Uptime Score +- Orders must be within **100bps from mid** to count +- Avg order size must be **≥ $2,000** + +**Minimum thresholds to receive rewards on a market:** +- Volume Score ≥ 1.5% +- Avg order size ≥ $2,000 +- At least 1% of total allocation + +**First month:** MM5 maker fees (best tier) regardless of volume. + +## Bot configuration + +Read the `backpack_perp_mm_deploy` skill before any deployment or config change. +The adapted aggressive config uses: +- Connector: `backpack_perpetual` +- **Position mode: ONEWAY** — universal default for all pmm_mister configs on any connector; HEDGE only if explicitly requested. Backpack specifically does not support HEDGE at all (errors on startup) +- 2 levels weighted 2:1, spreads 6bps/15bps +- 300s effectivization, 30s cooldown +- LIMIT_MAKER orders (type 3) for maker credit +- 2x leverage, 5% global SL +- $20K total_amount_quote per controller (min for $2K+ orders) + +## How you answer +- Lead with the key number or recommendation +- Use key: value format, not prose paragraphs +- Always cite the data source and timestamp +- Flag when data is estimated vs fetched live +- If a routine is available, call it and reason over its output — don't guess + +## Routines +- `backpack_volume_scan` — 24h perp volume scan with MM program thresholds diff --git a/agents/backpack_mm/routines/backpack_binance_funding_comparison.py b/agents/backpack_mm/routines/backpack_binance_funding_comparison.py new file mode 100644 index 000000000..b1b9473a9 --- /dev/null +++ b/agents/backpack_mm/routines/backpack_binance_funding_comparison.py @@ -0,0 +1,257 @@ +"""Compare funding rates between Backpack and Binance perpetuals — interval-normalized carry comparison.""" + +import asyncio +import logging + +import aiohttp +from pydantic import BaseModel, Field +from telegram.ext import ContextTypes + +from condor.reports import ReportBuilder +from routines.base import RoutineResult + +logger = logging.getLogger(__name__) + +CATEGORY = "Arbitrage" + +BACKPACK_API = "https://api.backpack.exchange/api/v1" +BINANCE_FAPI = "https://fapi.binance.com/fapi/v1" + +HOURS_PER_YEAR = 8760 + + +class Config(BaseModel): + """Compare Backpack vs Binance perpetual funding rates, normalized by each exchange's payment interval.""" + + backpack_interval_h: int = Field( + default=1, description="Backpack funding interval in hours (default: 1h)" + ) + binance_interval_h: int = Field( + default=8, description="Binance funding interval in hours (default: 8h)" + ) + min_spread_apr_pct: float = Field( + default=10.0, description="Min annualized spread % to highlight as opportunity" + ) + top_n: int = Field(default=20, description="Max rows in opportunities table") + + +def _normalize_base(base: str) -> str: + """Strip kilo-denomination prefix for cross-exchange matching (1000SHIB / kSHIB → SHIB).""" + if base.startswith("1000"): + return base[4:] + if base and base[0] == "k" and len(base) > 1 and base[1].isupper(): + return base[1:] + return base + + +async def _fetch_backpack_rates( + session: aiohttp.ClientSession, +) -> dict[str, tuple[float, str]]: + """Returns {canonical_base: (rate_per_payment, raw_base)} for all Backpack perp markets.""" + async with session.get(f"{BACKPACK_API}/markets") as resp: + markets = await resp.json() + + perp_symbols = [m["symbol"] for m in markets if m.get("marketType") == "PERP"] + + sem = asyncio.Semaphore(8) + + async def _fetch_one(symbol: str): + async with sem: + try: + async with session.get( + f"{BACKPACK_API}/fundingRates", + params={"symbol": symbol, "limit": 1}, + ) as r: + data = await r.json() + if isinstance(data, list) and data: + rate = data[0].get("fundingRate") + if rate is not None: + return symbol, float(rate) + except Exception as e: + logger.warning(f"Backpack funding fetch failed for {symbol}: {e}") + return symbol, None + + results = await asyncio.gather(*[_fetch_one(s) for s in perp_symbols]) + + out: dict[str, tuple[float, str]] = {} + for symbol, rate in results: + if rate is not None: + raw_base = symbol.replace("_USDC_PERP", "").replace("_USDT_PERP", "") + canonical = _normalize_base(raw_base) + out[canonical] = (rate, raw_base) + + return out + + +async def _fetch_binance_rates(session: aiohttp.ClientSession) -> dict[str, float]: + """Returns {canonical_base: rate_per_payment} for all Binance USDT perp markets.""" + async with session.get(f"{BINANCE_FAPI}/premiumIndex") as resp: + data = await resp.json() + + out: dict[str, float] = {} + for item in data: + symbol = item.get("symbol", "") + if not symbol.endswith("USDT"): + continue + raw_base = symbol[:-4] + canonical = _normalize_base(raw_base) + rate = item.get("lastFundingRate") + if rate is not None: + out[canonical] = float(rate) + + return out + + +async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: + async with aiohttp.ClientSession() as session: + bp_raw, bn_raw = await asyncio.gather( + _fetch_backpack_rates(session), + _fetch_binance_rates(session), + ) + + common = sorted(set(bp_raw) & set(bn_raw)) + + def to_apr(rate_per_payment: float, interval_h: int) -> float: + """Convert a per-payment rate to annualized rate (APR).""" + payments_per_year = HOURS_PER_YEAR / interval_h + return rate_per_payment * payments_per_year + + rows = [] + for base in common: + bp_rate_raw, raw_base = bp_raw[base] + bn_rate_raw = bn_raw[base] + + bp_apr = to_apr(bp_rate_raw, config.backpack_interval_h) + bn_apr = to_apr(bn_rate_raw, config.binance_interval_h) + spread_apr = bp_apr - bn_apr + + # Per-hour rates for a granular comparison column + bp_per_h_bps = (bp_rate_raw / config.backpack_interval_h) * 10000 + bn_per_h_bps = (bn_rate_raw / config.binance_interval_h) * 10000 + spread_per_h_bps = bp_per_h_bps - bn_per_h_bps + + if spread_apr > 0.0001: + trade = "Long BN / Short BP" + elif spread_apr < -0.0001: + trade = "Long BP / Short BN" + else: + trade = "Neutral" + + rows.append( + { + "Asset": base, + "BP Symbol": raw_base, + "BP APR": f"{bp_apr * 100:.2f}%", + "BN APR": f"{bn_apr * 100:.2f}%", + "Spread APR": f"{spread_apr * 100:+.2f}%", + "Spread bps/h": f"{spread_per_h_bps:+.3f}", + "Trade": trade, + "_spread_apr": spread_apr, + "_spread_abs_apr": abs(spread_apr), + "_bp_apr": bp_apr, + "_bn_apr": bn_apr, + "_spread_per_h_bps": spread_per_h_bps, + } + ) + + rows.sort(key=lambda r: r["_spread_abs_apr"], reverse=True) + + min_spread = config.min_spread_apr_pct / 100 + opps = [r for r in rows if r["_spread_abs_apr"] >= min_spread] + top_rows = opps[: config.top_n] + + table_cols = [ + "Asset", + "BP Symbol", + "BP APR", + "BN APR", + "Spread APR", + "Spread bps/h", + "Trade", + ] + clean_top = [{k: r[k] for k in table_cols} for r in top_rows] + clean_all = [{k: r[k] for k in table_cols} for r in rows[:60]] + + # ── Report ───────────────────────────────────────────────────────────── + builder = ReportBuilder("Backpack vs Binance Funding (Interval-Normalized)") + builder.source("routine", "backpack_binance_funding_comparison") + builder.tags(["funding", "arbitrage", "backpack", "binance", "carry"]) + builder.manual_order() + + builder.section("01 / OVERVIEW") + builder.kpi("Backpack Perps", str(len(bp_raw))) + builder.kpi("Binance Perps", str(len(bn_raw))) + builder.kpi("Common Markets", str(len(common))) + builder.kpi(f"Opps ≥{config.min_spread_apr_pct}% APR", str(len(opps))) + builder.kpi("BP Interval", f"{config.backpack_interval_h}h") + builder.kpi("BN Interval", f"{config.binance_interval_h}h") + + builder.markdown( + f"Rates are **annualized**: Backpack raw rate ÷ {config.backpack_interval_h}h × 8,760 h/yr; " + f"Binance raw rate ÷ {config.binance_interval_h}h × 8,760 h/yr. " + f"**Spread bps/h** = (BP per-hour rate − BN per-hour rate) × 10,000 — " + f"the carry you earn or pay per hour of holding the delta-neutral pair." + ) + + dataset_rows = [ + { + "asset": r["Asset"], + "bp_apr_pct": round(r["_bp_apr"] * 100, 2), + "bn_apr_pct": round(r["_bn_apr"] * 100, 2), + "spread_apr_pct": round(r["_spread_apr"] * 100, 2), + "abs_spread_apr_pct": round(r["_spread_abs_apr"] * 100, 2), + "spread_bps_per_h": round(r["_spread_per_h_bps"], 3), + } + for r in rows + ] + builder.dataset("funding", dataset_rows) + builder.chart( + "bar", + "Spread (APR %) by Asset", + "funding", + "asset", + "spread_apr_pct", + y_label="Spread APR (%)", + x_label="Asset", + reference_lines=[{"axis": "y", "value": 0, "label": "Zero"}], + ) + + builder.section( + "02 / OPPORTUNITIES", + f"Sorted by |spread APR|. Threshold: {config.min_spread_apr_pct}% APR. " + f"BP pays every {config.backpack_interval_h}h, BN every {config.binance_interval_h}h.", + ) + if clean_top: + builder.table(clean_top, table_cols) + else: + builder.markdown( + f"No common markets with |spread| ≥ {config.min_spread_apr_pct}% APR." + ) + + builder.section("03 / ALL COMMON MARKETS", "Full list sorted by |spread APR|.") + builder.table(clean_all, table_cols) + + report_id = await builder.save() + + # ── Inline summary ────────────────────────────────────────────────────── + top3 = top_rows[:3] + lines = [ + f"Backpack vs Binance Funding (BP={config.backpack_interval_h}h / BN={config.binance_interval_h}h intervals)", + f"{len(common)} common markets | {len(opps)} opps ≥{config.min_spread_apr_pct}% APR", + "", + ] + if top3: + for r in top3: + lines.append( + f" {r['Asset']:8s} BP={r['BP APR']:>9s} BN={r['BN APR']:>9s}" + f" spread={r['Spread APR']:>9s} → {r['Trade']}" + ) + else: + lines.append(" No opportunities above threshold.") + lines.append(f"\nReport: {report_id}") + + return RoutineResult( + text="\n".join(lines), + table_data=clean_top[:10], + table_columns=table_cols, + ) diff --git a/agents/backpack_mm/routines/backpack_funding_compare.py b/agents/backpack_mm/routines/backpack_funding_compare.py new file mode 100644 index 000000000..e69de29bb diff --git a/agents/backpack_mm/routines/backpack_mm_volume_analysis.py b/agents/backpack_mm/routines/backpack_mm_volume_analysis.py new file mode 100644 index 000000000..e69de29bb diff --git a/agents/backpack_mm/routines/backpack_volume_scan.py b/agents/backpack_mm/routines/backpack_volume_scan.py new file mode 100644 index 000000000..5d7373437 --- /dev/null +++ b/agents/backpack_mm/routines/backpack_volume_scan.py @@ -0,0 +1,257 @@ +"""Scan Backpack Exchange perp markets: 24h volume, MM program thresholds, and tractability ranking.""" + +import logging + +import aiohttp +from pydantic import BaseModel, Field +from telegram.ext import ContextTypes + +from condor.reports import ReportBuilder +from routines.base import RoutineResult + +logger = logging.getLogger(__name__) + +CATEGORY = "Market Data" + +BACKPACK_API = "https://api.backpack.exchange/api/v1" + + +class Config(BaseModel): + """Backpack perp volume scan — MM program threshold analysis""" + + target_share_pct: float = Field( + default=0.5, description="Target maker volume share % (e.g. 0.5 = 0.5%)" + ) + min_volume_score_pct: float = Field( + default=1.5, description="Min Volume Score % to qualify on a market" + ) + + +async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: + async with aiohttp.ClientSession() as session: + # Fetch tickers and markets in parallel + async with session.get(f"{BACKPACK_API}/tickers") as resp: + if resp.status != 200: + return f"Backpack API error: {resp.status}" + tickers = await resp.json() + + async with session.get(f"{BACKPACK_API}/markets") as resp: + if resp.status != 200: + return f"Backpack markets API error: {resp.status}" + markets = await resp.json() + + # Build market info lookup + market_info = {m["symbol"]: m for m in markets if m.get("marketType") == "PERP"} + + # Filter perp tickers and parse volumes + perps = [] + for t in tickers: + sym = t.get("symbol", "") + if not sym.endswith("_PERP"): + continue + quote_vol = float(t.get("quoteVolume", 0)) + last_price = float(t.get("lastPrice", 0)) + trades = int(t.get("trades", 0)) + price_change_pct = float(t.get("priceChangePercent", 0)) * 100 + base = sym.replace("_USDC_PERP", "").replace("k", "") + + # Get margin requirements from market info + info = market_info.get(sym, {}) + imf_base = float(info.get("imfFunction", {}).get("base", 0)) + max_leverage = round(1 / imf_base, 1) if imf_base > 0 else 0 + + perps.append( + { + "symbol": sym, + "base": base, + "quote_volume_24h": quote_vol, + "last_price": last_price, + "trades_24h": trades, + "price_change_pct": round(price_change_pct, 2), + "max_leverage": max_leverage, + } + ) + + if not perps: + return "No perpetual markets found on Backpack" + + # Sort by volume descending + perps.sort(key=lambda x: x["quote_volume_24h"], reverse=True) + + total_volume = sum(p["quote_volume_24h"] for p in perps) + + # Calculate thresholds + target_share = config.target_share_pct / 100 + min_vol_score = config.min_volume_score_pct / 100 + + target_total_volume = total_volume * target_share # 0.5% of total perp volume + monthly_target = target_total_volume * 30 + + # Per-market analysis + table_data = [] + for p in perps: + vol = p["quote_volume_24h"] + vol_share = (vol / total_volume * 100) if total_volume > 0 else 0 + # Volume needed for 1.5% Volume Score on this market + needed_for_qual = vol * min_vol_score / (1 - min_vol_score) + # Volume needed for target share on this market + needed_for_target = vol * target_share / (1 - target_share) + + table_data.append( + { + "Market": p["base"], + "24h Vol": f"${vol:,.0f}", + "Vol Share": f"{vol_share:.1f}%", + "Trades": f"{p['trades_24h']:,}", + "Price": ( + f"${p['last_price']:,.4f}" + if p["last_price"] < 1 + else f"${p['last_price']:,.2f}" + ), + "24h Chg": f"{p['price_change_pct']:+.1f}%", + "Max Lev": f"{p['max_leverage']:.0f}x", + "Need 1.5% VS": f"${needed_for_qual:,.0f}", + "Need 0.5% Total": f"${needed_for_target:,.0f}", + "_vol": vol, + "_needed_qual": needed_for_qual, + "_needed_target": needed_for_target, + } + ) + + # Build report + builder = ReportBuilder("Backpack Perp Volume Scan") + builder.source("routine", "backpack_volume_scan") + builder.tags(["backpack", "volume", "market-making", "perps"]) + builder.manual_order() + + builder.section("01 / PROGRAM OVERVIEW") + builder.kpi("Total Perp Volume (24h)", f"${total_volume:,.0f}") + builder.kpi("Perp Markets", str(len(perps))) + builder.kpi( + f"Target Share ({config.target_share_pct}%)", f"${target_total_volume:,.0f}/day" + ) + builder.kpi("Monthly Target (30d)", f"${monthly_target:,.0f}") + builder.kpi("Min Vol Score Threshold", f"{config.min_volume_score_pct}%") + + builder.section("02 / VOLUME BY MARKET") + builder.markdown( + "**Need 1.5% VS** = maker volume needed on that market to hit the 1.5% Volume Score minimum.\n\n" + f"**Need {config.target_share_pct}% Total** = maker volume needed on that market alone to reach " + f"{config.target_share_pct}% of total perp maker volume." + ) + + # Interactive dataset + dataset_rows = [] + for p in perps: + vol = p["quote_volume_24h"] + vol_share = (vol / total_volume * 100) if total_volume > 0 else 0 + needed_qual = vol * min_vol_score / (1 - min_vol_score) + needed_target = vol * target_share / (1 - target_share) + dataset_rows.append( + { + "market": p["base"], + "volume_24h": round(vol, 2), + "vol_share_pct": round(vol_share, 2), + "trades_24h": p["trades_24h"], + "last_price": p["last_price"], + "change_pct": p["price_change_pct"], + "max_leverage": p["max_leverage"], + "need_1_5_pct_vs": round(needed_qual, 2), + "need_target_share": round(needed_target, 2), + } + ) + + builder.dataset("perps", dataset_rows) + + builder.chart( + "bar", + "24h Volume by Market (USD)", + "perps", + "market", + "volume_24h", + y_label="Volume (USD)", + x_label="Market", + ) + + # Display table (clean columns without internal fields) + display_cols = [ + "Market", + "24h Vol", + "Vol Share", + "Trades", + "Price", + "24h Chg", + "Max Lev", + "Need 1.5% VS", + f"Need {config.target_share_pct}% Total", + ] + clean_table = [{k: row[k] for k in display_cols} for row in table_data] + builder.table(clean_table, display_cols) + + builder.section("03 / TRACTABILITY RANKING") + builder.markdown( + "Markets sorted by **lowest maker volume needed** to hit 1.5% Volume Score — " + "these are the easiest to qualify on." + ) + + # Sort by needed_for_qual ascending (most tractable first) + tractable = sorted(table_data, key=lambda x: x["_needed_qual"]) + tract_table = [] + for i, row in enumerate(tractable[:15], 1): + tract_table.append( + { + "#": i, + "Market": row["Market"], + "24h Vol": row["24h Vol"], + "Need for 1.5% VS": row["Need 1.5% VS"], + "Daily @ Target": row[f"Need {config.target_share_pct}% Total"], + } + ) + builder.table( + tract_table, ["#", "Market", "24h Vol", "Need for 1.5% VS", "Daily @ Target"] + ) + + builder.section("04 / STRATEGY NOTES") + builder.markdown( + f"**Reward pool**: $200k/month for perps\n\n" + f"**Blended Score** = (Volume Score × 0.8) + (Liquidity Score × 0.2)\n\n" + f"**Volume Score** = your maker vol / total maker vol on that market\n\n" + f"**Liquidity Score** = (Avg Order Size / Avg Spread) × Uptime\n\n" + f"**Minimum thresholds per market**: Volume Score ≥ 1.5%, Avg order ≥ $2,000\n\n" + f"**Tier Score** = max(Total Maker Share, Adjusted Maker Share) — " + f"lower-liquidity markets get multipliers in the adjusted calculation, " + f"making them more rewarding per dollar traded.\n\n" + f"**First month**: MM5 maker fees (best tier) regardless of volume." + ) + + report_id = await builder.save() + + # Build inline summary + top5 = perps[:5] + top5_lines = "\n".join( + f" {p['base']}: ${p['quote_volume_24h']:,.0f} (need ${p['quote_volume_24h'] * min_vol_score / (1 - min_vol_score):,.0f} for 1.5% VS)" + for p in top5 + ) + easiest3 = sorted(perps, key=lambda x: x["quote_volume_24h"])[:3] + easiest_lines = "\n".join( + f" {p['base']}: need only ${p['quote_volume_24h'] * min_vol_score / (1 - min_vol_score):,.0f}/day" + for p in easiest3 + ) + + summary = ( + f"Backpack Perp Volume Scan\n" + f"Total 24h perp volume: ${total_volume:,.0f}\n" + f"Target ({config.target_share_pct}% share): ${target_total_volume:,.0f}/day\n\n" + f"Top 5 by volume:\n{top5_lines}\n\n" + f"Easiest to qualify (1.5% VS):\n{easiest_lines}\n\n" + f"Report: {report_id}" + ) + + return RoutineResult( + text=summary, + table_data=[ + {k: row[k] for k in ["Market", "24h Vol", "Vol Share", "Need 1.5% VS"]} + for row in table_data[:10] + ], + table_columns=["Market", "24h Vol", "Vol Share", "Need 1.5% VS"], + ) diff --git a/agents/backpack_mm/skills/backpack_perp_mm_deploy/SKILL.md b/agents/backpack_mm/skills/backpack_perp_mm_deploy/SKILL.md new file mode 100644 index 000000000..71f6619c8 --- /dev/null +++ b/agents/backpack_mm/skills/backpack_perp_mm_deploy/SKILL.md @@ -0,0 +1,94 @@ +--- +name: backpack_perp_mm_deploy +description: End-to-end playbook for deploying pmm_mister bots on Backpack perps optimized + for the MM Rewards Program — adapted aggressive config, volume targets, and program + compliance. +when_to_use: When deploying, configuring, or tuning market making bots on Backpack + perpetuals. Also when asked about Backpack MM program compliance, order sizing, + or spread settings for the rewards program. +created: '2026-08-10T21:14:30Z' +source: agent:backpack_mm +--- + +# Backpack Perp MM Deploy Playbook + +Adapted from the market_making_expert's aggressive pmm_mister profile, +tuned specifically for the **Backpack MM Rewards Program** on perpetuals. + +## Program constraints (hard requirements) + +- **Avg order size ≥ $2,000** — orders below this don't count +- **Orders within 100bps from mid** — spreads must be < 0.01 +- **Volume Score ≥ 1.5%** per market to receive rewards +- **Post-only orders** — must be LIMIT_MAKER (type 3) for maker volume +- **Uptime matters** — Liquidity Score = (Avg Order Size / Avg Spread) × Uptime + +## Config template — Backpack MM Aggressive + +Fetch the full config from the companion file: +``` +manage_skill(action="read_file", name="backpack_perp_mm_deploy", file="config_backpack_aggressive.md") +``` + +### Key adaptations vs standard aggressive profile + +| Parameter | Standard Aggressive | Backpack MM | Why | +|-----------|-------------------|-------------|-----| +| total_amount_quote | $500 | **$20,000** | Min $2K orders need larger capital | +| levels | 2 equal | **2 weighted (2:1)** | Front-load L1 for volume, L2 for depth/Liquidity Score | +| spreads | 8bps, 15bps | **6bps, 15bps** | Tighter L1 for fills, wide L2 for depth | +| effectivization_time | 60s | **300s** | More time for TP to fill on thin markets → volume cycling | +| take_profit | 8bps | **6bps** | Fast cycle, viable with MM5 fees (first month) | +| cooldown_time | 30s | **30s** | Prevent adverse stacking on thin markets | +| leverage | 10x | **2x** | Mid-tier perps are volatile, protect capital | +| order_type | 3 | **3** | LIMIT_MAKER mandatory for program credit | + +## Deployment steps + +1. **Run volume scan** — `manage_routines(action="run", name="backpack_volume_scan")` + to get fresh 24h volume data and identify tractable markets. + +2. **Pick markets** — Select 3-5 mid-tier perps where needed maker volume for + 1.5% VS is achievable ($8-15K/day range). Avoid BTC (needs $1M+/day). + +3. **Verify pairs exist on Backpack perps** — Before creating any config, confirm + each ticker actually exists as a perpetual on Backpack by checking + `GET https://api.backpack.exchange/api/v1/markets`. The volume scan may report + spot-style tickers that don't match the actual perp symbol. Known gotchas: + - BONK perp = **kBONK**, SHIB perp = **kSHIB** (kilo-denominated) + - TRX, STABLE — **no perp exists** + - When in doubt, fetch the markets list and grep for the symbol + +4. **Read config template** — fetch `config_backpack_aggressive.md` companion file. + +5. **Adapt per market** — substitute `trading_pair`, verify leverage is within + the market's max (check `imfFunction.base` from /api/v1/markets). + +6. **Set position mode** — Backpack perpetual **only supports ONEWAY** position + mode. Always set `"position_mode": "ONEWAY"`. Never use `HEDGE` — the bot + will error on startup if HEDGE is set. + +7. **Upsert configs** — one config per market via `manage_controllers(action="upsert", target="config", ...)`. + Name convention: `NNN_pmm_backpack_`. + +8. **Deploy single bot** with all controllers: + ``` + manage_bots(action="deploy", bot_name="backpack_mm_perps_", + controllers_config=["config1", "config2", "config3"]) + ``` + +9. **Verify** — `manage_bots(action="status")` to confirm all controllers running. + +## Volume math (per controller) + +- Order size L1: $20K × 0.25 × (2/3) = **$3,333** ✓ (above $2K) +- Order size L2: $20K × 0.25 × (1/3) = **$1,667** (depth, may not count for avg) +- Per complete fill→TP cycle: 2 × $3,333 = **$6,666** maker volume +- Need ~2-3 cycles/day per market for 1.5% VS on $900K market +- At 300s effectivization + 30s cooldown: achievable + +## Monitoring + +- Schedule `backpack_volume_scan` daily to track volume shifts +- Check bot PnL and fill rates via mm_bot_report +- Watch for markets dropping below threshold — may need to rotate diff --git a/agents/backpack_mm/skills/backpack_perp_mm_deploy/config_backpack_aggressive.md b/agents/backpack_mm/skills/backpack_perp_mm_deploy/config_backpack_aggressive.md new file mode 100644 index 000000000..9f7790b19 --- /dev/null +++ b/agents/backpack_mm/skills/backpack_perp_mm_deploy/config_backpack_aggressive.md @@ -0,0 +1,94 @@ +# Backpack MM Aggressive Config Template + +Adapted from the MM expert's aggressive profile, tuned for Backpack MM Rewards +Program compliance. All orders are LIMIT_MAKER (post-only) for maker credit. + +## Config (substitute connector/pair/amount per market) + +```json +{ + "controller_type": "generic", + "controller_name": "pmm_mister", + "connector_name": "backpack_perpetual", + "trading_pair": "-USDC", + "total_amount_quote": 20000, + "portfolio_allocation": 0.25, + "target_base_pct": 0.5, + "min_base_pct": 0.3, + "max_base_pct": 0.7, + "buy_spreads": "0.0003,0.0015", + "sell_spreads": "0.0003,0.0015", + "buy_amounts_pct": "2,1", + "sell_amounts_pct": "2,1", + "executor_refresh_time": 20, + "buy_cooldown_time": 30, + "sell_cooldown_time": 30, + "buy_position_effectivization_time": 300, + "sell_position_effectivization_time": 300, + "price_distance_tolerance": 0.0005, + "refresh_tolerance": 0.0003, + "tolerance_scaling": 1.1, + "open_order_type": 3, + "take_profit": 0.0006, + "take_profit_order_type": 3, + "leverage": 2, + "position_mode": "ONEWAY", + "position_side": "BUY", + "max_active_executors_by_level": 2, + "tick_mode": false, + "min_skew": 1.0, + "global_tp_enabled": false, + "global_sl_enabled": true, + "global_stop_loss": 0.05, + "global_sl_activation_from": "target_base", + "global_pnl_reference": "position" +} +``` + +## Position mode — universal default + +**Always use `"position_mode": "ONEWAY"`** for pmm_mister configs on any connector. +This is the universal default — not just a Backpack restriction. HEDGE should only +be used when the user explicitly requests it. + +On Backpack specifically, HEDGE is not supported at all and will error on startup. +On other connectors (Hyperliquid, Binance, etc.), ONEWAY is still the default unless +the user says otherwise. + +## Parameter rationale + +**Order sizing** (meets $2K minimum): +- Level 1: $20K × 0.25 × (2/3) = $3,333 per order +- Level 2: $20K × 0.25 × (1/3) = $1,667 per order (depth) +- Both sides quoting = continuous presence + +**Spreads** (within 100bps program limit): +- L1: 3bps — very tight, maximizes fill rate and volume generation +- L2: 15bps — passive depth, helps Liquidity Score + +**Effectivization** (300s): +- After a fill, the LIMIT_MAKER TP order stays on book for 5 minutes +- On mid-tier perps, price needs time to cycle back for TP fill +- Complete fill→TP cycle = 2× notional in maker volume +- 60s was too short (most positions expired to hold); 300s catches ~50%+ of TPs + +**Take profit** (6bps): +- With MM5 fees (~0.02% maker), round-trip ~0.04% +- 6bps TP gives 1.5× fee coverage — tight but viable +- If Backpack fees are higher than expected, bump to 8bps + +**Leverage** (2x): +- Mid-tier perps (ZEC, PUMP, WLD) can gap 5-10% +- At 2x: 5% move = 10% drawdown → survivable with 5% global SL +- At 5x: 5% move = 25% drawdown → dangerous + +**Cooldown** (30s): +- Prevents stacking fills at similar prices on thin markets +- 15s was too aggressive for $500-900K/day volume markets + +## Adjustments per market + +- **PAXG**: Gold-backed, low vol → can keep 3bps, reduce cooldown to 20s +- **ZEC/WLD**: Moderate vol → use template as-is +- **PUMP**: Higher vol, can gap → monitor closely with 3bps, widen to 5bps if adverse fills spike +- **Leverage**: Always verify max leverage per market (1/imfFunction.base from API) diff --git a/agents/backpack_mm/strategies/backpack_perp_mm/strategy.md b/agents/backpack_mm/strategies/backpack_perp_mm/strategy.md new file mode 100644 index 000000000..35c9c1b0c --- /dev/null +++ b/agents/backpack_mm/strategies/backpack_perp_mm/strategy.md @@ -0,0 +1,69 @@ +--- +name: Backpack Perp MM +description: '' +agent_key: null +skills: [] +default_config: + frequency_sec: 3600 + total_amount_quote: 60000 +default_trading_context: '' +created_by: 481175164 +created_at: '2026-08-10T21:16:55.078013+00:00' +--- + +# Backpack Perp MM Strategy + +Market making on Backpack perpetuals targeting the MM Rewards Program. +Agent pinned to `local` server. + +## Loop playbook + +Each tick: + +1. **Volume check** — Run `backpack_volume_scan` routine to get fresh 24h volumes. + Compare current market volumes against previous tick. Flag markets where volume + dropped below qualification threshold. + +2. **Bot health** — Check `manage_bots(action="status")` for the running bot. + Verify all controllers are in `running` state. Check for errors in logs. + +3. **Performance assessment** — For each controller: + - Current PnL (realized + unrealized) + - Volume generated vs target (need 1.5% VS per market) + - Inventory balance (base_pct within min/max band) + - Fill rate and TP hit rate + +4. **Compliance check** — Verify program metrics: + - Are avg order sizes ≥ $2,000? + - Are spreads within 100bps? + - Is uptime acceptable for Liquidity Score? + +5. **Adjust if needed**: + - If a market's volume dropped and qualification is at risk → consider rotating + to a more tractable market + - If inventory is drifting → adjust target/min/max_base_pct live + - If PnL is bleeding → widen spreads or increase cooldown + - If volume target is not being met → tighten spreads cautiously + +6. **Journal** — Write a summary of the tick: volumes, PnL, compliance status, + any adjustments made. + +## Active deployment + +- Bot: `backpack_mm_perps_20260811-20260810-211539` +- Controllers (all 3bps L1 / 15bps L2, 6bps TP, 2x leverage, $20K each): + - `001_pmm_backpack_ZEC-USDC` — ZEC perp + - `002_pmm_backpack_WLD-USDC` — WLD perp + - `003_pmm_backpack_PUMP-USDC` — PUMP perp + +## Risk limits + +- Max $20K per controller ($60K total across 3 markets) +- 2x leverage +- 5% global stop loss per controller +- No global TP — let volume cycle + +## Skills to read + +- `backpack_perp_mm_deploy` — full config template and deployment steps +- `backpack_volume_scan` routine — fresh volume data diff --git a/agents/brigado/AGENT.md b/agents/brigado/AGENT.md new file mode 100644 index 000000000..b81b6a9d5 --- /dev/null +++ b/agents/brigado/AGENT.md @@ -0,0 +1,16 @@ +--- +name: Brigado +description: BRL market making research agent. Tracks USDT-BRL spreads, BTC/fiat + volume dynamics, and bot performance for the Brazilian market making operation. +agent_key: claude-acp:sonnet +tools: [] +when_to_consult: '' +server_required: true +created_by: 481175164 +created_at: '2026-05-04T00:00:00+00:00' +--- + +## Brigado + +### Objective +Research and monitoring agent for the Brazilian market making operation. Provides dashboards, volume analysis, and bot performance tracking focused on BRL pairs and fiat market dynamics. diff --git a/agents/brigado/routines/bot_report.py b/agents/brigado/routines/bot_report.py new file mode 100644 index 000000000..561257bea --- /dev/null +++ b/agents/brigado/routines/bot_report.py @@ -0,0 +1,1752 @@ +"""Bot Performance Report — Clean 2-message report with USD values and rebate tracking.""" + +CATEGORY = "Bot Analysis" + +import asyncio +import logging +import os +import re +from collections import defaultdict +from datetime import datetime, timezone + +import plotly.graph_objects as go +from plotly.subplots import make_subplots +from pydantic import BaseModel, Field +from telegram.ext import ContextTypes + +from config_manager import get_client, get_config_manager + +logger = logging.getLogger(__name__) + +REBATE_RATE = 0.00015 # 0.015% of volume — only applies to BRL-quoted pairs + +# Config analysis constants +_CONFIG_SKIP_PARAMS = { + "id", + "controller_type", + "controller_name", + "manual_kill_switch", + "_config_name", +} +_CONFIG_KEY_PARAMS = { + "buy_spreads", + "sell_spreads", + "buy_amounts_pct", + "sell_amounts_pct", + "total_amount_quote", + "stop_loss", + "take_profit", + "time_limit", + "executor_refresh_time", + "cooldown_time", + "leverage", + "dca_spreads", + "dca_amounts", + "trailing_stop", + "executor_activation_bounds", + "top_executor_refresh_time", + "position_rebalance_threshold_pct", +} + + +def _pair_has_rebate(pair: str, quote_currency: str) -> bool: + """Rebate only applies to pairs quoted in the rebate currency (BRL).""" + if not pair or "-" not in pair: + return False + return pair.split("-")[-1].upper() == quote_currency.upper() + + +class Config(BaseModel): + """Generate a formatted performance report for all active bots.""" + + quote_currency: str = Field( + default="BRL", + description="Quote currency of the bots (for USD conversion)", + ) + target_chat_id: int | None = Field( + default=None, + description="Send report to this chat ID instead of the current chat (e.g. a group chat)", + ) + server_name: str = Field( + default="brigado_2", + description="Server to connect to for bot data (bots run on this server)", + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _fmt(n: float, decimals: int = 2) -> str: + if abs(n) >= 1_000: + return f"{n:,.{decimals}f}" + return f"{n:.{decimals}f}" + + +def _sign(n: float) -> str: + return "+" if n >= 0 else "-" + + +def _emoji(n: float) -> str: + return "🟢" if n > 0 else "🔴" if n < 0 else "⚪" + + +def _md_escape(text: str) -> str: + """Escape special characters for Telegram MarkdownV2.""" + special = r"_*[]()~`>#+-=|{}.!" + result = [] + for ch in text: + if ch in special: + result.append(f"\\{ch}") + else: + result.append(ch) + return "".join(result) + + +# --------------------------------------------------------------------------- +# Data fetching +# --------------------------------------------------------------------------- + + +async def _get_bots_data(client) -> dict: + data = await client.bot_orchestration.get_active_bots_status() + if isinstance(data, dict) and "data" in data: + return data["data"] + return {} + + +async def _get_bot_runs(client) -> dict: + runs = {} + try: + data = await client.bot_orchestration.get_bot_runs() + if isinstance(data, dict) and "data" in data: + for run in data["data"]: + if run.get("deployment_status") == "DEPLOYED" and run.get( + "deployed_at" + ): + runs[run["bot_name"]] = run["deployed_at"] + except Exception: + pass + return runs + + +async def _get_prices(client, connector: str, pairs: list[str]) -> dict[str, float]: + try: + result = await client.market_data.get_prices( + connector_name=connector, trading_pairs=",".join(pairs) + ) + return result.get("prices", {}) if result else {} + except Exception as e: + logger.warning(f"Failed to get prices: {e}") + return {} + + +async def _get_balances(client) -> dict: + """Fetch portfolio balances from all accounts.""" + try: + return await client.portfolio.get_state(refresh=False) + except Exception as e: + logger.warning(f"Failed to get balances: {e}") + return {} + + +async def _get_market_volume_since( + client, connector: str, pair: str, since_dt: datetime | None +) -> tuple[float, float]: + """Return (base_volume, quote_volume) accumulated from candles since *since_dt*. + + Uses 1h candles spanning from *since_dt* to now so the market volume window + matches the bot's runtime, giving an accurate market-share calculation. + Falls back to a single 1d candle when *since_dt* is unavailable. + """ + try: + if since_dt: + now = datetime.now(timezone.utc) + days = max(1, int((now - since_dt).total_seconds() / 86400) + 1) + candles = await client.market_data.get_candles_last_days( + connector_name=connector, + trading_pair=pair, + days=days, + interval="1h", + ) + else: + candles = await client.market_data.get_candles( + connector_name=connector, + trading_pair=pair, + interval="1d", + max_records=1, + ) + + if candles: + data = candles if isinstance(candles, list) else candles.get("data", []) + if not data: + return 0.0, 0.0 + + since_ts = since_dt.timestamp() if since_dt else 0 + total_base = 0.0 + total_quote = 0.0 + for row in data: + # Only count candles that start at or after the deploy time + ts = float(row.get("timestamp", 0) or 0) + # Normalise ms → s if needed + if ts > 1e12: + ts /= 1000 + if ts >= since_ts: + total_base += float(row.get("volume", 0) or 0) + total_quote += float(row.get("quote_asset_volume", 0) or 0) + return total_base, total_quote + except Exception as e: + logger.warning(f"Failed to get volume for {pair}: {e}") + return 0.0, 0.0 + + +# --------------------------------------------------------------------------- +# Chart builders (Plotly) +# --------------------------------------------------------------------------- + +_DARK_BG = "#1a1a2e" +_DARK_PAPER = "#16213e" +_GREEN = "#00d4aa" +_RED = "#ff6b6b" +_BLUE = "#58a6ff" +_YELLOW = "#ffdd57" +_COLORS = [ + "#58a6ff", + "#00d4aa", + "#ff6b6b", + "#ffdd57", + "#c084fc", + "#fb923c", + "#38bdf8", + "#f472b6", +] + + +def _chart_layout(title: str, **kwargs) -> dict: + base = dict( + title=dict(text=title, font=dict(color="#e6edf3", size=14), x=0.5), + paper_bgcolor=_DARK_PAPER, + plot_bgcolor=_DARK_BG, + font=dict(color="#ccc", size=11), + margin=dict(l=60, r=80, t=50, b=80), + width=1100, + height=450, + ) + base.update(kwargs) + return base + + +def _quote_is_usd(pair: str) -> bool: + """Check if a pair's quote currency is a USD stablecoin.""" + if not pair or "-" not in pair: + return False + return pair.split("-")[-1].upper() in ("USD", "USDT", "USDC", "BUSD", "FDUSD") + + +def _vol_to_usd(ctrl: dict, to_usd: float) -> float: + """Convert controller volume to USD based on the pair's quote currency.""" + if _quote_is_usd(ctrl.get("pair", "")): + return ctrl["vol"] + return ctrl["vol"] * to_usd + + +def _amount_quote_to_usd(ctrl: dict, to_usd: float) -> float: + """Convert total_amount_quote to USD based on the pair's quote currency.""" + amt = ctrl.get("total_amount_quote", 0) + if _quote_is_usd(ctrl.get("pair", "")): + return amt + return amt * to_usd + + +def _market_vol_to_usd( + pair: str, + vol_base: float, + vol_quote: float, + to_usd: float, + prices: dict[str, float], +) -> float: + """Convert market candle volume to USD for any pair.""" + base = pair.split("-")[0].upper() if "-" in pair else "" + quote = pair.split("-")[-1].upper() if "-" in pair else "" + + # If base is a stablecoin, vol_base is already ~USD + if base in ("USDT", "USDC", "BUSD", "DAI", "FDUSD"): + return vol_base + + # If quote is USD stablecoin, vol_quote is already USD + if quote in ("USD", "USDT", "USDC", "BUSD", "FDUSD"): + return ( + vol_quote + if vol_quote > 0 + else vol_base * (prices.get(f"{base}-USDT", 0) or 0) + ) + + # Non-USD quote (e.g. BRL) — convert quote volume, or use base price + if vol_quote > 0: + return vol_quote * to_usd + bp = prices.get(f"{base}-USDT", 0) or prices.get(f"{base}-USD", 0) + return vol_base * bp if bp else 0 + + +def _short_chart_name(ctrl_id: str, pair: str = "") -> str: + """Clean controller name for chart labels.""" + name = ctrl_id + # Strip common prefixes + name = re.sub(r"^mm_", "", name) + name = re.sub(r"pmm_?mister_?", "", name) + # Strip connector prefixes (binance_, binance_perpetual_, etc.) + name = re.sub(r"^(binance|okx|bybit|kucoin)(_perpetual)?_", "", name) + # Strip pair fragment + if pair: + pair_frag = pair.replace("-", "_") + name = name.replace(pair_frag, "").replace(pair, "") + while "__" in name: + name = name.replace("__", "_") + name = name.strip("_") + name = re.sub(r"config[_]?(\d+)", r"c\1", name) + return name or ctrl_id[:20] + + +def _build_pnl_chart( + controllers: list[dict], + to_usd: float, + quote_currency: str = "BRL", + ctrl_configs: dict | None = None, +) -> go.Figure: + """Horizontal bar chart: PnL breakdown, Total PnL, and Vol/Capital by controller.""" + # Sort by net PnL + sorted_ctrls = sorted(controllers, key=lambda c: c["net"] * to_usd) + names = [_short_chart_name(c["id"], c["pair"]) for c in sorted_ctrls] + realized = [c["realized"] * to_usd for c in sorted_ctrls] + unrealized = [c["unrealized"] * to_usd for c in sorted_ctrls] + rebates = [ + ( + c["vol"] * to_usd * REBATE_RATE + if _pair_has_rebate(c["pair"], quote_currency) + else 0 + ) + for c in sorted_ctrls + ] + totals = [r + u + rb for r, u, rb in zip(realized, unrealized, rebates)] + volumes = [_vol_to_usd(c, to_usd) for c in sorted_ctrls] + + # Capital from configs (converted to USD) + has_capital = False + capitals = [] + if ctrl_configs: + for c in sorted_ctrls: + cfg = ctrl_configs.get(c["id"], {}) + amt = float(cfg.get("total_amount_quote", 0) or 0) + ctrl_for_conv = {"total_amount_quote": amt, "pair": c["pair"]} + capitals.append(_amount_quote_to_usd(ctrl_for_conv, to_usd)) + if amt > 0: + has_capital = True + + # Vol/Capital turnover ratio + vol_cap_ratios = [] + if has_capital: + for v, cap in zip(volumes, capitals): + vol_cap_ratios.append(v / cap if cap > 0 else 0) + + ncols = 3 if has_capital else 2 + col_widths = [0.40, 0.30, 0.30] if has_capital else [0.55, 0.45] + subtitles = ( + ["PnL Breakdown (USD)", "Total PnL (USD)", "Vol / Capital"] + if has_capital + else ["PnL Breakdown (USD)", "Total PnL (USD)"] + ) + + fig = make_subplots( + rows=1, + cols=ncols, + shared_yaxes=True, + column_widths=col_widths, + horizontal_spacing=0.03, + subplot_titles=subtitles, + ) + + # Col 1: PnL breakdown (stacked) + fig.add_trace( + go.Bar( + y=names, + x=realized, + orientation="h", + name="Realized", + marker_color=_GREEN, + opacity=0.85, + ), + row=1, + col=1, + ) + fig.add_trace( + go.Bar( + y=names, + x=unrealized, + orientation="h", + name="Unrealized", + marker_color=_BLUE, + opacity=0.85, + ), + row=1, + col=1, + ) + fig.add_trace( + go.Bar( + y=names, + x=rebates, + orientation="h", + name="Rebate", + marker_color=_YELLOW, + opacity=0.7, + ), + row=1, + col=1, + ) + + # Col 2: Total PnL (single bar per controller) + total_colors = [_GREEN if t >= 0 else _RED for t in totals] + fig.add_trace( + go.Bar( + y=names, + x=totals, + orientation="h", + name="Total", + marker_color=total_colors, + opacity=0.9, + text=[f"{'+'if t>=0 else ''}{t:.1f}" for t in totals], + textposition="inside", + textfont=dict(size=10, color="white"), + insidetextanchor="middle", + showlegend=False, + ), + row=1, + col=2, + ) + + # Col 3: Vol/Capital turnover + if has_capital: + fig.add_trace( + go.Bar( + y=names, + x=vol_cap_ratios, + orientation="h", + name="Vol/Capital", + marker_color="#8b5cf6", + opacity=0.85, + text=[f"{r:.0f}x" if r > 0 else "-" for r in vol_cap_ratios], + textposition="outside", + textfont=dict(size=10), + showlegend=False, + ), + row=1, + col=3, + ) + + height = max(350, len(names) * 40 + 100) + fig.update_layout( + **_chart_layout("PnL Breakdown · Total · Vol/Capital", height=height), + barmode="relative", + legend=dict(orientation="h", y=-0.12, x=0.5, xanchor="center"), + ) + fig.update_xaxes(gridcolor="#2a2a4a", title="USD", row=1, col=1) + fig.update_xaxes(gridcolor="#2a2a4a", title="USD", row=1, col=2) + if has_capital: + fig.update_xaxes(gridcolor="#2a2a4a", title="Turnover (x)", row=1, col=3) + fig.update_yaxes(gridcolor="#2a2a4a", row=1, col=1) + # Fix subplot title colors + for ann in fig.layout.annotations: + ann.font = dict(color="#aaa", size=11) + return fig + + +def _build_pair_overview_chart( + controllers: list[dict], + to_usd: float, + volumes_since: dict[str, tuple[float, float]], + prices: dict[str, float], + quote_currency: str = "BRL", +) -> go.Figure: + """Unified horizontal bar chart: PnL, Volume, and Market Share by pair.""" + # Aggregate data per pair + pair_data: dict[str, dict] = defaultdict( + lambda: {"realized": 0, "unrealized": 0, "vol": 0} + ) + for c in controllers: + key = c["pair"] or "unknown" + pair_data[key]["realized"] += c["realized"] * to_usd + pair_data[key]["unrealized"] += c["unrealized"] * to_usd + pair_data[key]["vol"] += _vol_to_usd(c, to_usd) + + # Sort pairs by total volume (highest at top) + pairs = sorted(pair_data.keys(), key=lambda p: pair_data[p]["vol"]) + + # Compute market share data + pair_market: dict[str, dict] = {} + for pair in pairs: + vol_data = volumes_since.get(pair, (0, 0)) + vol_base, vol_quote = vol_data if isinstance(vol_data, tuple) else (vol_data, 0) + mkt_usd = ( + _market_vol_to_usd(pair, vol_base, vol_quote, to_usd, prices) + if (vol_base > 0 or vol_quote > 0) + else 0 + ) + our_vol = pair_data[pair]["vol"] + share = (our_vol / mkt_usd * 100) if mkt_usd > 0 else 0 + pair_market[pair] = {"market": mkt_usd, "ours": our_vol, "share": share} + + has_market = any(d["market"] > 0 for d in pair_market.values()) + ncols = 3 if has_market else 2 + col_widths = [0.4, 0.35, 0.25] if has_market else [0.5, 0.5] + subtitles = ( + ["PnL (USD)", "Volume (USD)", "Market Share"] + if has_market + else ["PnL (USD)", "Volume (USD)"] + ) + + fig = make_subplots( + rows=1, + cols=ncols, + shared_yaxes=True, + column_widths=col_widths, + horizontal_spacing=0.04, + subplot_titles=subtitles, + ) + + # Col 1: PnL (realized + unrealized + rebate) + fig.add_trace( + go.Bar( + y=pairs, + x=[pair_data[p]["realized"] for p in pairs], + orientation="h", + name="Realized", + marker_color=_GREEN, + opacity=0.85, + ), + row=1, + col=1, + ) + fig.add_trace( + go.Bar( + y=pairs, + x=[pair_data[p]["unrealized"] for p in pairs], + orientation="h", + name="Unrealized", + marker_color=_BLUE, + opacity=0.85, + ), + row=1, + col=1, + ) + fig.add_trace( + go.Bar( + y=pairs, + x=[ + ( + pair_data[p]["vol"] * REBATE_RATE + if _pair_has_rebate(p, quote_currency) + else 0 + ) + for p in pairs + ], + orientation="h", + name="Rebate", + marker_color=_YELLOW, + opacity=0.7, + ), + row=1, + col=1, + ) + + # Col 2: Volume + fig.add_trace( + go.Bar( + y=pairs, + x=[pair_data[p]["vol"] for p in pairs], + orientation="h", + name="Volume", + marker_color="#8b5cf6", + opacity=0.8, + ), + row=1, + col=2, + ) + + # Col 3: Market Share as percentage bar + if has_market: + shares = [pair_market[p]["share"] for p in pairs] + fig.add_trace( + go.Bar( + y=pairs, + x=shares, + orientation="h", + name="Market Share", + marker_color=[_BLUE if s > 0 else "#30363d" for s in shares], + opacity=0.85, + text=[f"{s:.2f}%" if s > 0 else "N/A" for s in shares], + textposition="outside", + textfont=dict(size=11, color=_BLUE), + hovertemplate="%{y}
Share: %{x:.2f}%
" + "Ours: $%{customdata[0]:,.0f}
" + "Market: $%{customdata[1]:,.0f}", + customdata=[ + [pair_market[p]["ours"], pair_market[p]["market"]] for p in pairs + ], + ), + row=1, + col=ncols, + ) + + height = max(350, len(pairs) * 55 + 120) + fig.update_layout( + **_chart_layout("Pair Overview: PnL · Volume · Market Share", height=height), + barmode="relative", + legend=dict(orientation="h", y=-0.12, x=0.5, xanchor="center"), + ) + for col in range(1, ncols + 1): + fig.update_xaxes(gridcolor="#2a2a4a", row=1, col=col) + if has_market: + fig.update_xaxes(title="%", row=1, col=ncols) + fig.update_yaxes(gridcolor="#2a2a4a", row=1, col=1) + # Fix subplot title colors + for ann in fig.layout.annotations: + ann.font = dict(color="#aaa", size=11) + return fig + + +# --------------------------------------------------------------------------- +# Config analysis helpers +# --------------------------------------------------------------------------- + + +def _cfg_fmt_value(v) -> str: + if v is None: + return "—" + if isinstance(v, bool): + return "✓" if v else "✗" + if isinstance(v, float): + if v < 0.001: + return f"{v:.5f}" + if v < 0.01: + return f"{v:.4f}" + return f"{v:.3f}" if v < 10 else f"{v:.1f}" + if isinstance(v, list): + return ", ".join(_cfg_fmt_value(x) for x in v) + if isinstance(v, dict): + return " | ".join(f"{k}={_cfg_fmt_value(vv)}" for k, vv in v.items()) + return str(v) + + +def _cfg_normalize(v): + if v is None: + return None + if isinstance(v, str): + try: + return float(v) + except ValueError: + return v.strip().lower() + if isinstance(v, list): + return [_cfg_normalize(x) for x in v] + if isinstance(v, dict): + return {k: _cfg_normalize(vv) for k, vv in v.items()} + return v + + +def _build_config_parallel_coords( + all_configs: dict, running_ids: set +) -> go.Figure | None: + NUMERIC_AXES = [ + ("total_amount_quote", "Capital", None), + ( + "buy_spreads", + "Buy Spread L1 %", + lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, + ), + ( + "sell_spreads", + "Sell Spread L1 %", + lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, + ), + ( + "buy_spreads", + "Buy Spread L2 %", + lambda v: ( + (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None + ), + ), + ( + "sell_spreads", + "Sell Spread L2 %", + lambda v: ( + (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None + ), + ), + ("executor_refresh_time", "Refresh Time", None), + ("cooldown_time", "Cooldown", None), + ("leverage", "Leverage", None), + ("stop_loss", "Stop Loss", None), + ("take_profit", "Take Profit", None), + ("time_limit", "Time Limit", None), + ("top_executor_refresh_time", "Top Refresh", None), + ( + "dca_spreads", + "DCA Spread L1", + lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, + ), + ( + "dca_amounts", + "DCA Amt L1", + lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, + ), + ( + "trailing_stop", + "Trail Stop", + lambda v: ( + (v.get("activation_price_delta") if isinstance(v, dict) else v) + if v + else None + ), + ), + ( + "executor_activation_bounds", + "Activation Bounds", + lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, + ), + ] + + rows = [] + labels = [] + for label, cfg in all_configs.items(): + clean_label = label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", "") + is_running = clean_label in running_ids or label.startswith("\U0001f7e2") + pair = cfg.get("trading_pair", "unknown") + row = {"_running": 1 if is_running else 0, "_pair": pair} + for param, axis_name, extractor in NUMERIC_AXES: + raw = cfg.get(param) + val = extractor(raw) if extractor else raw + try: + row[axis_name] = float(val) if val is not None else None + except (TypeError, ValueError): + row[axis_name] = None + rows.append(row) + labels.append(_short_chart_name(clean_label, pair)) + + if len(rows) < 2: + return None + + valid_axes = [] + for _, axis_name, _ in NUMERIC_AXES: + vals = [r[axis_name] for r in rows if r[axis_name] is not None] + if len(vals) >= 2 and len(set(vals)) > 1: + valid_axes.append(axis_name) + + if len(valid_axes) < 2: + return None + + unique_pairs = sorted(set(r["_pair"] for r in rows)) + pair_to_idx = {p: i for i, p in enumerate(unique_pairs)} + color_values = [pair_to_idx[r["_pair"]] for r in rows] + + n_pairs = len(unique_pairs) + if n_pairs == 1: + colorscale = [[0, _BLUE], [1, _BLUE]] + else: + colorscale = [ + [i / (n_pairs - 1), _COLORS[i % len(_COLORS)]] for i in range(n_pairs) + ] + + for axis_name in valid_axes: + vals = [r[axis_name] for r in rows if r[axis_name] is not None] + fill = min(vals) if vals else 0 + for r in rows: + if r[axis_name] is None: + r[axis_name] = fill + + dimensions = [] + for axis_name in valid_axes: + vals = [r[axis_name] for r in rows] + if "%" in axis_name: + vals = [v * 100 for v in vals] + dimensions.append( + dict( + label=axis_name, + values=vals, + range=( + [min(vals), max(vals)] + if min(vals) != max(vals) + else [min(vals) - 1, max(vals) + 1] + ), + ) + ) + + fig = go.Figure( + data=go.Parcoords( + line=dict(color=color_values, colorscale=colorscale, showscale=False), + dimensions=dimensions, + customdata=labels, + ) + ) + for i, pair in enumerate(unique_pairs): + fig.add_trace( + go.Scatter( + x=[None], + y=[None], + mode="markers", + marker=dict(size=10, color=_COLORS[i % len(_COLORS)]), + name=pair, + showlegend=True, + ) + ) + fig.update_layout( + **_chart_layout( + f"Config Parameter Space ({len(rows)} configs, {len(valid_axes)} axes)", + width=1400, + height=600, + ), + legend=dict( + orientation="h", y=-0.05, x=0.5, xanchor="center", font=dict(size=12) + ), + ) + return fig + + +def _build_config_spread_chart(all_configs: dict) -> go.Figure | None: + def _to_list(v) -> list: + if v is None: + return [] + if isinstance(v, (int, float)): + return [float(v)] + if isinstance(v, list): + return [float(x) for x in v] + return [] + + items = [] + for label, cfg in all_configs.items(): + buy = _to_list(cfg.get("buy_spreads")) + sell = _to_list(cfg.get("sell_spreads")) + if buy or sell: + short = _short_chart_name( + label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", ""), + cfg.get("trading_pair", ""), + ) + items.append( + { + "name": short, + "pair": cfg.get("trading_pair", ""), + "buy_l1": buy[0] * 100 if len(buy) >= 1 else 0, + "sell_l1": sell[0] * 100 if len(sell) >= 1 else 0, + "buy_l2": buy[1] * 100 if len(buy) >= 2 else 0, + "sell_l2": sell[1] * 100 if len(sell) >= 2 else 0, + } + ) + + if not items: + return None + + items.sort(key=lambda c: (c["pair"], c["name"])) + names = [c["name"] for c in items] + + fig = go.Figure() + fig.add_trace( + go.Bar( + x=names, + y=[c["buy_l1"] for c in items], + name="Buy L1", + marker_color=_GREEN, + opacity=0.9, + ) + ) + fig.add_trace( + go.Bar( + x=names, + y=[c["sell_l1"] for c in items], + name="Sell L1", + marker_color=_RED, + opacity=0.9, + ) + ) + if any(c["buy_l2"] > 0 or c["sell_l2"] > 0 for c in items): + fig.add_trace( + go.Bar( + x=names, + y=[c["buy_l2"] for c in items], + name="Buy L2", + marker_color=_GREEN, + opacity=0.5, + ) + ) + fig.add_trace( + go.Bar( + x=names, + y=[c["sell_l2"] for c in items], + name="Sell L2", + marker_color=_RED, + opacity=0.5, + ) + ) + + fig.update_layout( + **_chart_layout("Spread Levels by Controller (%)", width=1200, height=500), + barmode="group", + yaxis=dict(gridcolor="#2a2a4a", title="Spread %"), + xaxis=dict(gridcolor="#2a2a4a", tickangle=-45), + legend=dict(orientation="h", y=1.1, x=0.5, xanchor="center"), + ) + fig.update_layout(margin=dict(l=60, r=40, t=60, b=120)) + return fig + + +def _build_config_capital_chart(all_configs: dict) -> go.Figure | None: + data = [] + for label, cfg in all_configs.items(): + amt = float(cfg.get("total_amount_quote", 0) or 0) + if amt <= 0: + continue + short = _short_chart_name( + label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", ""), + cfg.get("trading_pair", ""), + ) + data.append({"name": short, "pair": cfg.get("trading_pair", ""), "amount": amt}) + + if not data: + return None + + data.sort(key=lambda d: (d["pair"], d["amount"])) + unique_pairs = list(dict.fromkeys(d["pair"] for d in data)) + pair_colors = {p: _COLORS[i % len(_COLORS)] for i, p in enumerate(unique_pairs)} + + fig = go.Figure() + for pair in unique_pairs: + pair_data = [d for d in data if d["pair"] == pair] + fig.add_trace( + go.Bar( + y=[d["name"] for d in pair_data], + x=[d["amount"] for d in pair_data], + orientation="h", + name=pair, + marker_color=pair_colors[pair], + opacity=0.85, + text=[f"{d['amount']:,.0f}" for d in pair_data], + textposition="auto", + ) + ) + + height = max(400, len(data) * 22 + 100) + fig.update_layout( + **_chart_layout( + "Capital Allocation by Controller (quote)", width=1200, height=height + ), + barmode="stack", + xaxis=dict(gridcolor="#2a2a4a", title="Amount (quote)"), + yaxis=dict(gridcolor="#2a2a4a"), + legend=dict(orientation="h", y=1.05, x=0.5, xanchor="center"), + ) + fig.update_layout(margin=dict(l=150, r=40, t=60, b=50)) + return fig + + +# --------------------------------------------------------------------------- +# Report builder +# --------------------------------------------------------------------------- + + +def _build_report( + bots_data: dict, + bot_runs: dict, + prices: dict[str, float], + volumes_since: dict[str, tuple[float, float]], + quote_usd_rate: float, + quote_currency: str = "BRL", + ctrl_configs: dict | None = None, + balances: dict | None = None, +) -> list[str]: + ctrl_configs = ctrl_configs or {} + + # ---- Parse controller data ---- + controllers = [] + + for bot_name, bot_data in bots_data.items(): + if isinstance(bot_data, str): + continue + + performance = {} + if isinstance(bot_data, dict): + performance = bot_data.get("performance", {}) + elif isinstance(bot_data, list): + performance = { + f"ctrl_{i}": {"performance": c} for i, c in enumerate(bot_data) + } + + for ctrl_name, ctrl_data in performance.items(): + if not isinstance(ctrl_data, dict): + continue + perf = ctrl_data.get("performance", ctrl_data) + cfg = ctrl_configs.get(ctrl_name, {}) + trading_pair = cfg.get("trading_pair", "") or perf.get("trading_pair", "") + + vol = float(perf.get("volume_traded", 0) or 0) + realized = float(perf.get("realized_pnl_quote", 0) or 0) + unrealized = float(perf.get("unrealized_pnl_quote", 0) or 0) + total_amount_quote = float(cfg.get("total_amount_quote", 0) or 0) + + controllers.append( + { + "id": ctrl_name, + "bot": bot_name, + "pair": trading_pair, + "vol": vol, + "realized": realized, + "unrealized": unrealized, + "net": realized + unrealized, + "total_amount_quote": total_amount_quote, + } + ) + + if not controllers: + return ["No controller data found\\."] + + # ---- Runtime ---- + earliest_dt = None + for bn in {c["bot"] for c in controllers}: + ts = bot_runs.get(bn) + if ts: + try: + ts_str = str(ts) + if ts_str.endswith("Z"): + ts_str = ts_str[:-1] + "+00:00" + dt = datetime.fromisoformat(ts_str) + if earliest_dt is None or dt < earliest_dt: + earliest_dt = dt + except (ValueError, TypeError): + pass + + now_dt = datetime.now(timezone.utc) + runtime_h = (now_dt - earliest_dt).total_seconds() / 3600 if earliest_dt else 0 + if earliest_dt: + start_str = earliest_dt.strftime("%b %d, %H:%M UTC") + else: + start_str = "N/A" + + # ---- Totals ---- + total_vol_q = sum(c["vol"] for c in controllers) + total_realized_q = sum(c["realized"] for c in controllers) + total_unrealized_q = sum(c["unrealized"] for c in controllers) + total_net_q = total_realized_q + total_unrealized_q + + to_usd = quote_usd_rate + total_vol = sum(_vol_to_usd(c, to_usd) for c in controllers) + total_realized = total_realized_q * to_usd + total_unrealized = total_unrealized_q * to_usd + total_net = total_net_q * to_usd + total_rebate_vol = sum( + _vol_to_usd(c, to_usd) + for c in controllers + if _pair_has_rebate(c["pair"], quote_currency) + ) + total_rebate = total_rebate_vol * REBATE_RATE + + vol_hr = total_vol / runtime_h if runtime_h > 0 else 0 + realized_hr = total_realized / runtime_h if runtime_h > 0 else 0 + rebate_hr = total_rebate / runtime_h if runtime_h > 0 else 0 + net_after_rebate = total_net + total_rebate + + # ---- Market volumes ---- + unique_pairs = {c["pair"] for c in controllers if c["pair"]} + market_lines = [] + for pair in sorted(unique_pairs): + vol_data = volumes_since.get(pair, (0, 0)) + vol_24h_base, vol_24h_quote = ( + vol_data if isinstance(vol_data, tuple) else (vol_data, 0) + ) + if vol_24h_base <= 0 and vol_24h_quote <= 0: + continue + mkt_usd = _market_vol_to_usd(pair, vol_24h_base, vol_24h_quote, to_usd, prices) + our_vol = sum(_vol_to_usd(c, to_usd) for c in controllers if c["pair"] == pair) + share = (our_vol / mkt_usd * 100) if mkt_usd > 0 else 0 + market_lines.append( + f" {_md_escape(pair)}: mkt \\${_md_escape(_fmt(mkt_usd))} " + f"\\| ours \\${_md_escape(_fmt(our_vol))} \\({_md_escape(f'{share:.2f}')}%\\)" + ) + + # ---- FX rate display ---- + qc = quote_currency.upper() + if qc not in ("USD", "USDT"): + fx_val = _fmt(1 / to_usd if to_usd > 0 else 0) + fx_display = f"1 USD \\= {_md_escape(fx_val)} {_md_escape(qc)}" + else: + fx_display = "" + + # ---- MSG 1: Overview ---- + num_bots = len({c["bot"] for c in controllers}) + num_ctrls = len(controllers) + + # ---- Balance summary ---- + balance_lines = [] + total_balance_usd = 0.0 + if balances: + token_totals: dict[str, float] = defaultdict(float) + token_values: dict[str, float] = defaultdict(float) + for account_data in balances.values(): + for connector_balances in account_data.values(): + if not isinstance(connector_balances, list): + continue + for b in connector_balances: + token = b.get("token", "") + value = float(b.get("value", 0) or 0) + units = float(b.get("units", 0) or 0) + if value > 0: + token_totals[token] += units + token_values[token] += value + total_balance_usd = sum(token_values.values()) + # Top tokens by value + sorted_tokens = sorted(token_values.items(), key=lambda x: x[1], reverse=True) + for token, value in sorted_tokens[:5]: + pct = (value / total_balance_usd * 100) if total_balance_usd > 0 else 0 + balance_lines.append( + f" {_md_escape(token)}: \\${_md_escape(_fmt(value))} \\({_md_escape(f'{pct:.0f}')}%\\)" + ) + + msg1_parts = [ + f"*📊 TRADING REPORT*", + f"━━━━━━━━━━━━━━━━━━━━━━━━", + f"", + f"⏱ {_md_escape(f'{runtime_h:.1f}')}h runtime \\(since {_md_escape(start_str)}\\)", + f"🤖 {num_bots} bots \\| {num_ctrls} controllers", + ] + if fx_display: + msg1_parts.append(f"💱 {fx_display}") + + if balance_lines: + msg1_parts += [ + f"", + f"💰 *Balance: \\${_md_escape(_fmt(total_balance_usd))}*", + ] + balance_lines + + msg1_parts += [ + f"", + f"{_emoji(total_net)} *Performance \\(USD\\)*", + f"Volume: \\${_md_escape(_fmt(total_vol))}", + f"Realized: {_md_escape(_sign(total_realized))}\\${_md_escape(_fmt(abs(total_realized)))}", + f"Unrealized: {_md_escape(_sign(total_unrealized))}\\${_md_escape(_fmt(abs(total_unrealized)))}", + f"Net PnL: {_md_escape(_sign(total_net))}\\${_md_escape(_fmt(abs(total_net)))}", + f"Rebate: \\+\\${_md_escape(_fmt(total_rebate))}", + f"Net\\+Rebate: {_md_escape(_sign(net_after_rebate))}\\${_md_escape(_fmt(abs(net_after_rebate)))}", + f"", + f"⏳ *Hourly Rates*", + f"Vol/hr: \\${_md_escape(_fmt(vol_hr))}", + f"Realized/hr: {_md_escape(_sign(realized_hr))}\\${_md_escape(_fmt(abs(realized_hr)))}", + f"Rebate/hr: \\+\\${_md_escape(_fmt(rebate_hr))}", + ] + + if market_lines: + msg1_parts += [f"", f"📈 *Market Share*"] + market_lines + + # ---- MSG 2: Controllers grouped by pair ---- + msg2_parts = [ + f"*📋 CONTROLLERS BY PAIR*", + f"━━━━━━━━━━━━━━━━━━━━━━━━", + ] + + pair_groups: dict[str, list[dict]] = defaultdict(list) + for c in controllers: + pair_groups[c["pair"] or "unknown"].append(c) + + sorted_pairs = sorted( + pair_groups.items(), key=lambda x: sum(c["vol"] for c in x[1]), reverse=True + ) + + for pair, ctrls in sorted_pairs: + grp_vol = sum(_vol_to_usd(c, to_usd) for c in ctrls) + grp_realized = sum(c["realized"] for c in ctrls) * to_usd + grp_unrealized = sum(c["unrealized"] for c in ctrls) * to_usd + grp_net = grp_realized + grp_unrealized + grp_rebate = ( + grp_vol * REBATE_RATE if _pair_has_rebate(pair, quote_currency) else 0 + ) + grp_adj = grp_net + grp_rebate + + pair_escaped = _md_escape(pair) + msg2_parts.append(f"") + msg2_parts.append( + f"▸ *{pair_escaped}* \\({len(ctrls)} ctrls\\) " + f"{_emoji(grp_adj)} {_md_escape(_sign(grp_adj))}\\${_md_escape(_fmt(abs(grp_adj)))}" + ) + + # Build controller table rows + sorted_ctrls = sorted(ctrls, key=lambda c: c["vol"], reverse=True) + + def _short_name(ctrl_id: str, pair: str) -> str: + name = ctrl_id + if name.startswith("mm_"): + name = name[3:] + # Strip all variations of pmm_mister prefix + name = re.sub(r"pmm_?mister_?", "", name) + pair_frag = pair.replace("-", "_") if pair else "" + if pair_frag: + name = name.replace(pair_frag, "").replace(pair, "") + while "__" in name: + name = name.replace("__", "_") + name = name.strip("_") + name = re.sub(r"config[_]?(\d+)", r"c\1", name) + return name or ctrl_id[:20] + + rows = [] + for c in sorted_ctrls: + vol_usd = _vol_to_usd(c, to_usd) + net_usd = c["net"] * to_usd + reb_usd = ( + vol_usd * REBATE_RATE if _pair_has_rebate(pair, quote_currency) else 0 + ) + adj_usd = net_usd + reb_usd + short = _short_name(c["id"], pair) + rows.append((short, vol_usd, net_usd, adj_usd)) + + # Calculate column widths + name_w = max(len(r[0]) for r in rows) + vol_strs = [f"${_fmt(r[1])}" for r in rows] + pnl_strs = [f"{_sign(r[2])}${_fmt(abs(r[2]))}" for r in rows] + adj_strs = [f"{_sign(r[3])}${_fmt(abs(r[3]))}" for r in rows] + + vol_w = max(len(s) for s in vol_strs) + pnl_w = max(len(s) for s in pnl_strs) + adj_w = max(len(s) for s in adj_strs) + + # Header + separator + rows inside a code block + hdr = ( + f"{'Name':<{name_w}} {'Vol':>{vol_w}} {'PnL':>{pnl_w}} {'Adj':>{adj_w}}" + ) + sep = "─" * len(hdr) + + table_lines = [hdr, sep] + for i, (short, vol_usd, net_usd, adj_usd) in enumerate(rows): + e = "+" if adj_usd >= 0 else "-" + table_lines.append( + f"{short:<{name_w}} {vol_strs[i]:>{vol_w}} {pnl_strs[i]:>{pnl_w}} {adj_strs[i]:>{adj_w}}" + ) + + # Pair subtotal row + sub_vol = f"${_fmt(grp_vol)}" + sub_pnl = f"{_sign(grp_net)}${_fmt(abs(grp_net))}" + sub_adj = f"{_sign(grp_adj)}${_fmt(abs(grp_adj))}" + table_lines.append(sep) + table_lines.append( + f"{'TOTAL':<{name_w}} {sub_vol:>{vol_w}} {sub_pnl:>{pnl_w}} {sub_adj:>{adj_w}}" + ) + + msg2_parts.append("```") + msg2_parts.extend(table_lines) + msg2_parts.append("```") + + return ["\n".join(msg1_parts), "\n".join(msg2_parts)] + + +# --------------------------------------------------------------------------- +# Telegram HTTP fallback (when context.bot is unavailable) +# --------------------------------------------------------------------------- + + +async def _send_telegram_http( + chat_id: int, text: str, parse_mode: str = "MarkdownV2" +) -> bool: + """Send a message via Telegram HTTP API directly. Used as fallback.""" + token = os.environ.get("TELEGRAM_BOT_TOKEN", "") + if not token: + logger.error("TELEGRAM_BOT_TOKEN not set, cannot send via HTTP fallback") + return False + try: + import httpx + + url = f"https://api.telegram.org/bot{token}/sendMessage" + payload = {"chat_id": chat_id, "text": text, "parse_mode": parse_mode} + async with httpx.AsyncClient(timeout=10) as http_client: + resp = await http_client.post(url, json=payload) + data = resp.json() + if data.get("ok"): + return True + # Retry without parse_mode if formatting fails + if "can't parse" in data.get("description", "").lower(): + payload.pop("parse_mode") + resp = await http_client.post(url, json=payload) + data = resp.json() + if data.get("ok"): + return True + logger.error(f"Telegram HTTP send failed: {data.get('description')}") + except Exception as e: + logger.error(f"Telegram HTTP fallback error: {e}") + return False + + +# --------------------------------------------------------------------------- +# Main entry point +# --------------------------------------------------------------------------- + + +async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: + chat_id = context._chat_id if hasattr(context, "_chat_id") else None + if not chat_id: + return "No chat_id available" + + # Auto-resolve target_chat_id from notes if not explicitly set + send_to = config.target_chat_id + if not send_to: + try: + from condor.preferences import get_note + + user_data = context.user_data if hasattr(context, "user_data") else {} + # Look up note for the active server's group chat + server_name = ( + config.server_name + or get_config_manager().get_chat_default_server(chat_id) + ) + if server_name: + note_val = get_note(user_data, f"server.{server_name}.group_chat_id") + if note_val: + send_to = int(note_val) + except Exception: + pass + if not send_to: + # Fallback to CONDOR_CHAT_ID env var (the invoking user's chat) + env_chat = os.environ.get("CONDOR_CHAT_ID", "") + send_to = int(env_chat) if env_chat else chat_id + + # Connect to the specified server (default: brigado_2 where bots run) + if config.server_name: + client = await get_config_manager().get_client(config.server_name) + else: + client = await get_client(chat_id, context=context) + if not client: + return "Could not connect to API server" + + bots_data, bot_runs, balances = await asyncio.gather( + _get_bots_data(client), + _get_bot_runs(client), + _get_balances(client), + ) + + if not bots_data: + await context.bot.send_message(chat_id=send_to, text="No active bots found.") + return "No active bots" + + # Fetch controller configs for trading_pair + ctrl_configs = {} + for bot_name in bots_data: + try: + configs = await client.controllers.get_bot_controller_configs(bot_name) + if isinstance(configs, list): + for cfg in configs: + cid = cfg.get("id") or cfg.get("controller_id", "") + if cid: + ctrl_configs[cid] = cfg + except Exception: + pass + + # Determine trading pairs and connector + trading_pairs = set() + connector = "binance" + for bot_data in bots_data.values(): + if not isinstance(bot_data, dict): + continue + for ctrl_name, ctrl_data in bot_data.get("performance", {}).items(): + if not isinstance(ctrl_data, dict): + continue + cfg = ctrl_configs.get(ctrl_name, {}) + perf = ctrl_data.get("performance", ctrl_data) + pair = cfg.get("trading_pair", "") or perf.get("trading_pair", "") + if pair: + trading_pairs.add(pair) + conn = cfg.get("connector_name", "") or perf.get("connector_name", "") + if conn: + connector = conn.split("_")[0] + + # Build price pairs for USD conversion + price_pairs = list(trading_pairs) + for bt in {p.split("-")[0] for p in trading_pairs if "-" in p}: + if f"{bt}-USDT" not in price_pairs: + price_pairs.append(f"{bt}-USDT") + for qt in {p.split("-")[-1] for p in trading_pairs if "-" in p}: + if qt not in ("USDT", "USD") and f"USDT-{qt}" not in price_pairs: + price_pairs.append(f"USDT-{qt}") + + prices = await _get_prices(client, connector, price_pairs) + + # FX rate fallback + qc = config.quote_currency.upper() + fx_pair = f"USDT-{qc}" if qc not in ("USD", "USDT") else "" + if fx_pair and not prices.get(fx_pair): + fx_prices = await _get_prices(client, "binance", [fx_pair]) + prices.update(fx_prices) + + # Market volumes since deploy + earliest_dt = None + for bn in bots_data: + ts = bot_runs.get(bn) + if ts: + try: + ts_str = str(ts) + if ts_str.endswith("Z"): + ts_str = ts_str[:-1] + "+00:00" + dt = datetime.fromisoformat(ts_str) + if earliest_dt is None or dt < earliest_dt: + earliest_dt = dt + except (ValueError, TypeError): + pass + + volumes_since = {} + vol_tasks = { + pair: _get_market_volume_since(client, connector, pair, earliest_dt) + for pair in trading_pairs + } + vol_results = await asyncio.gather(*vol_tasks.values()) + for pair, vol_tuple in zip(vol_tasks.keys(), vol_results): + volumes_since[pair] = vol_tuple + + # Quote → USD rate + quote_usd_rate = 1.0 + if qc not in ("USD", "USDT"): + usdt_quote = prices.get(f"USDT-{qc}", 0) + if usdt_quote and usdt_quote > 0: + quote_usd_rate = 1.0 / usdt_quote + else: + quote_usdt = prices.get(f"{qc}-USDT", 0) + if quote_usdt and quote_usdt > 0: + quote_usd_rate = quote_usdt + + # Build and send + messages = _build_report( + bots_data, + bot_runs, + prices, + volumes_since, + quote_usd_rate, + quote_currency=qc, + ctrl_configs=ctrl_configs, + balances=balances, + ) + + sent = 0 + for i, msg in enumerate(messages): + if i > 0: + await asyncio.sleep(1) + for attempt in range(3): + try: + await context.bot.send_message( + chat_id=send_to, + text=msg, + parse_mode="MarkdownV2", + ) + sent += 1 + break + except Exception as e: + err = str(e).lower() + if "flood control" in err or "retry in" in err: + match = re.search(r"retry in (\d+)", err) + wait = int(match.group(1)) + 1 if match else 10 + await asyncio.sleep(wait) + else: + logger.error(f"Failed to send msg {i} via bot: {e}") + # Fallback: send via HTTP directly to Telegram API + if await _send_telegram_http(send_to, msg): + sent += 1 + break + + # Strip MarkdownV2 escaping for plain-text return + def _strip_md(text: str) -> str: + return text.replace("\\", "") + + plain_report = "\n\n".join(_strip_md(m) for m in messages) + + from condor.reports import ReportBuilder + + builder = ReportBuilder("Trading Report") + builder.source("routine", "bot_report").tags(["bots", "performance"]) + builder.manual_order() + + # ---- Parse controller data for charts ---- + controllers_parsed = [] + for bot_name, bot_data in bots_data.items(): + if isinstance(bot_data, str): + continue + performance = {} + if isinstance(bot_data, dict): + performance = bot_data.get("performance", {}) + elif isinstance(bot_data, list): + performance = { + f"ctrl_{i}": {"performance": c} for i, c in enumerate(bot_data) + } + for ctrl_name, ctrl_data in performance.items(): + if not isinstance(ctrl_data, dict): + continue + perf = ctrl_data.get("performance", ctrl_data) + cfg = ctrl_configs.get(ctrl_name, {}) + trading_pair = cfg.get("trading_pair", "") or perf.get("trading_pair", "") + vol = float(perf.get("volume_traded", 0) or 0) + realized = float(perf.get("realized_pnl_quote", 0) or 0) + unrealized = float(perf.get("unrealized_pnl_quote", 0) or 0) + total_amount_quote = float(cfg.get("total_amount_quote", 0) or 0) + controllers_parsed.append( + { + "id": ctrl_name, + "bot": bot_name, + "pair": trading_pair, + "vol": vol, + "realized": realized, + "unrealized": unrealized, + "net": realized + unrealized, + "total_amount_quote": total_amount_quote, + } + ) + + # ---- Compute totals for KPIs ---- + total_vol_usd = sum(_vol_to_usd(c, quote_usd_rate) for c in controllers_parsed) + total_realized_usd = sum(c["realized"] for c in controllers_parsed) * quote_usd_rate + total_unrealized_usd = ( + sum(c["unrealized"] for c in controllers_parsed) * quote_usd_rate + ) + total_net_usd = total_realized_usd + total_unrealized_usd + rebate_vol_usd = sum( + _vol_to_usd(c, quote_usd_rate) + for c in controllers_parsed + if _pair_has_rebate(c["pair"], qc) + ) + total_rebate_usd = rebate_vol_usd * REBATE_RATE + net_after_rebate_usd = total_net_usd + total_rebate_usd + num_bots = len({c["bot"] for c in controllers_parsed}) + num_ctrls = len(controllers_parsed) + + # Runtime + earliest_deploy = None + for bn in {c["bot"] for c in controllers_parsed}: + ts = bot_runs.get(bn) + if ts: + try: + ts_str = str(ts) + if ts_str.endswith("Z"): + ts_str = ts_str[:-1] + "+00:00" + dt = datetime.fromisoformat(ts_str) + if earliest_deploy is None or dt < earliest_deploy: + earliest_deploy = dt + except (ValueError, TypeError): + pass + now_utc = datetime.now(timezone.utc) + runtime_h = ( + (now_utc - earliest_deploy).total_seconds() / 3600 if earliest_deploy else 0 + ) + + # Balance + total_balance_usd = 0.0 + if balances: + for account_data in balances.values(): + for connector_balances in account_data.values(): + if not isinstance(connector_balances, list): + continue + for b in connector_balances: + total_balance_usd += float(b.get("value", 0) or 0) + + # ---- KPI cards ---- + pnl_trend = "up" if net_after_rebate_usd >= 0 else "down" + builder.kpi( + "Net PnL + Rebate", + f"{'+'if net_after_rebate_usd>=0 else ''}${_fmt(abs(net_after_rebate_usd))}", + delta=f"${_fmt(abs(net_after_rebate_usd / runtime_h) if runtime_h else 0)}/hr", + trend=pnl_trend, + ) + total_capital_usd = sum( + _amount_quote_to_usd(c, quote_usd_rate) for c in controllers_parsed + ) + builder.kpi( + "Volume", + f"${_fmt(total_vol_usd)}", + delta=f"${_fmt(total_vol_usd / runtime_h if runtime_h else 0)}/hr", + ) + builder.kpi( + "Rebate", + f"${_fmt(total_rebate_usd)}", + delta=f"${_fmt(total_rebate_usd / runtime_h if runtime_h else 0)}/hr", + trend="up", + ) + if total_capital_usd > 0: + builder.kpi( + "Capital Deployed", + f"${_fmt(total_capital_usd)}", + delta=f"{num_ctrls} controllers", + ) + builder.kpi("Balance", f"${_fmt(total_balance_usd)}") + builder.kpi( + "Runtime", + f"{runtime_h:.1f}h", + delta=f"{num_bots} bots, {num_ctrls} controllers", + ) + + # ---- Charts ---- + if controllers_parsed: + builder.plotly( + _build_pair_overview_chart( + controllers_parsed, quote_usd_rate, volumes_since, prices, qc + ) + ) + builder.plotly( + _build_pnl_chart(controllers_parsed, quote_usd_rate, qc, ctrl_configs) + ) + + # ---- Controller details table ---- + ctrl_rows = [] + for c in sorted( + controllers_parsed, key=lambda x: x["net"] * quote_usd_rate, reverse=True + ): + vol_usd = _vol_to_usd(c, quote_usd_rate) + net_usd = c["net"] * quote_usd_rate + reb_usd = vol_usd * REBATE_RATE if _pair_has_rebate(c["pair"], qc) else 0 + adj_usd = net_usd + reb_usd + cap_usd = _amount_quote_to_usd(c, quote_usd_rate) + ctrl_rows.append( + { + "Controller": _short_chart_name(c["id"], c["pair"]), + "Pair": c["pair"], + "Capital": f"${_fmt(cap_usd)}" if cap_usd > 0 else "-", + "Volume": f"${_fmt(vol_usd)}", + "Realized": f"{'+'if c['realized']*quote_usd_rate>=0 else '-'}${_fmt(abs(c['realized']*quote_usd_rate))}", + "Unrealized": f"{'+'if c['unrealized']*quote_usd_rate>=0 else '-'}${_fmt(abs(c['unrealized']*quote_usd_rate))}", + "Rebate": f"+${_fmt(reb_usd)}", + "Total": f"{'+'if adj_usd>=0 else '-'}${_fmt(abs(adj_usd))}", + } + ) + if ctrl_rows: + builder.table(ctrl_rows) + + # ---- Pair summary table ---- + pair_groups: dict[str, list[dict]] = defaultdict(list) + for c in controllers_parsed: + pair_groups[c["pair"] or "unknown"].append(c) + pair_rows = [] + for pair, ctrls in sorted( + pair_groups.items(), key=lambda x: sum(c["vol"] for c in x[1]), reverse=True + ): + grp_vol = sum(_vol_to_usd(c, quote_usd_rate) for c in ctrls) + grp_realized = sum(c["realized"] for c in ctrls) * quote_usd_rate + grp_unrealized = sum(c["unrealized"] for c in ctrls) * quote_usd_rate + grp_net = grp_realized + grp_unrealized + grp_rebate = grp_vol * REBATE_RATE if _pair_has_rebate(pair, qc) else 0 + grp_adj = grp_net + grp_rebate + # Market share + vol_data = volumes_since.get(pair, (0, 0)) + vol_base, vol_quote = vol_data if isinstance(vol_data, tuple) else (vol_data, 0) + mkt_usd = _market_vol_to_usd(pair, vol_base, vol_quote, quote_usd_rate, prices) + share = f"{grp_vol / mkt_usd * 100:.2f}%" if mkt_usd > 0 else "N/A" + grp_capital = sum(_amount_quote_to_usd(c, quote_usd_rate) for c in ctrls) + pair_rows.append( + { + "Pair": pair, + "Controllers": str(len(ctrls)), + "Capital": f"${_fmt(grp_capital)}" if grp_capital > 0 else "-", + "Volume": f"${_fmt(grp_vol)}", + "Realized": f"{'+'if grp_realized>=0 else '-'}${_fmt(abs(grp_realized))}", + "Net PnL": f"{'+'if grp_net>=0 else '-'}${_fmt(abs(grp_net))}", + "Rebate": f"+${_fmt(grp_rebate)}", + "Adjusted": f"{'+'if grp_adj>=0 else '-'}${_fmt(abs(grp_adj))}", + "Mkt Share": share, + } + ) + if pair_rows: + builder.table(pair_rows) + + # ---- Balance table ---- + if balances: + token_totals: dict[str, float] = defaultdict(float) + token_values: dict[str, float] = defaultdict(float) + for account_data in balances.values(): + for connector_balances in account_data.values(): + if not isinstance(connector_balances, list): + continue + for b in connector_balances: + token = b.get("token", "") + value = float(b.get("value", 0) or 0) + units = float(b.get("units", 0) or 0) + if value > 0: + token_totals[token] += units + token_values[token] += value + if token_values: + bal_rows = [] + for token, value in sorted( + token_values.items(), key=lambda x: x[1], reverse=True + ): + pct = (value / total_balance_usd * 100) if total_balance_usd > 0 else 0 + bal_rows.append( + { + "Token": token, + "Amount": _fmt(token_totals[token], 4), + "Value (USD)": f"${_fmt(value)}", + "Weight": f"{pct:.1f}%", + } + ) + builder.table(bal_rows) + + # ---- Config Analysis section ---- + if len(ctrl_configs) > 1: + all_params = set() + for cfg in ctrl_configs.values(): + all_params.update(cfg.keys()) + all_params -= _CONFIG_SKIP_PARAMS + sorted_params = sorted( + all_params, key=lambda p: (0 if p in _CONFIG_KEY_PARAMS else 1, p) + ) + + # Group configs by pair + cfg_pair_groups: dict[str, dict] = defaultdict(dict) + for cid, cfg in ctrl_configs.items(): + pair = cfg.get("trading_pair", "unknown") + cfg_pair_groups[pair][cid] = cfg + + # Count total diffs + total_diffs = 0 + pair_sections = [] + for pair, pair_cfgs in sorted( + cfg_pair_groups.items(), key=lambda x: -len(x[1]) + ): + if len(pair_cfgs) < 2: + continue + labels = list(pair_cfgs.keys()) + shorts = [_short_chart_name(l, pair) for l in labels] + + diff_rows = [] + for param in sorted_params: + values = [_cfg_normalize(pair_cfgs[l].get(param)) for l in labels] + non_none = [v for v in values if v is not None] + if len(non_none) < 2 or len(set(str(v) for v in non_none)) <= 1: + continue + is_key = param in _CONFIG_KEY_PARAMS + row = {"Parameter": f"{'🔴 ' if is_key else '🟡 '}{param}"} + for short, label in zip(shorts, labels): + row[short] = _cfg_fmt_value(pair_cfgs[label].get(param)) + diff_rows.append(row) + + if diff_rows: + total_diffs += len(diff_rows) + pair_sections.append((pair, len(pair_cfgs), diff_rows)) + + if pair_sections: + builder.markdown( + "## Config Analysis\n\nParameter differences across running controllers. 🔴 = key tuning param, 🟡 = other." + ) + for pair, n_cfgs, rows in pair_sections: + builder.markdown( + f"### {pair} — {len(rows)} diffs across {n_cfgs} controllers" + ) + builder.table(rows) + + # Config charts + labeled_configs = {f"\U0001f7e2 {k}": v for k, v in ctrl_configs.items()} + running_ids = set(ctrl_configs.keys()) + + fig_pc = _build_config_parallel_coords(labeled_configs, running_ids) + if fig_pc: + if not pair_sections: + builder.markdown("## Config Analysis") + builder.plotly(fig_pc) + + fig_spread = _build_config_spread_chart(labeled_configs) + if fig_spread: + builder.plotly(fig_spread) + + await builder.save() + + return f"Report sent: {sent}/{len(messages)} messages\n\n{plain_report}" diff --git a/agents/brigado/routines/brl_dashboard.py b/agents/brigado/routines/brl_dashboard.py new file mode 100644 index 000000000..691844004 --- /dev/null +++ b/agents/brigado/routines/brl_dashboard.py @@ -0,0 +1,573 @@ +"""Multi-timeframe USDT-BRL technical dashboard with Plotly chart.""" + +CATEGORY = "Market Data" + +import asyncio +import io +import logging +from datetime import datetime, timezone + +import numpy as np +import pandas as pd +import plotly.graph_objects as go +from plotly.subplots import make_subplots +from pydantic import BaseModel, Field +from telegram.ext import ContextTypes + +from config_manager import get_client +from routines.base import RoutineResult + +logger = logging.getLogger(__name__) + + +class Config(BaseModel): + """USDT-BRL multi-timeframe dashboard: intraday, weekly, monthly, quarterly.""" + + trading_pair: str = Field(default="USDT-BRL", description="Trading pair") + connector_name: str = Field(default="binance", description="Exchange connector") + + +# --------------------------------------------------------------------------- +# Indicators +# --------------------------------------------------------------------------- + + +def compute_sma(series: pd.Series, period: int) -> pd.Series: + return series.rolling(window=period, min_periods=period).mean() + + +def compute_ema(series: pd.Series, period: int) -> pd.Series: + return series.ewm(span=period, adjust=False).mean() + + +def compute_rsi(series: pd.Series, period: int = 14) -> pd.Series: + delta = series.diff() + gain = delta.where(delta > 0, 0.0) + loss = (-delta).where(delta < 0, 0.0) + avg_gain = gain.ewm(alpha=1 / period, min_periods=period).mean() + avg_loss = loss.ewm(alpha=1 / period, min_periods=period).mean() + rs = avg_gain / avg_loss.replace(0, np.nan) + rsi = 100 - (100 / (1 + rs)) + return rsi.fillna(50) + + +def find_sr_levels( + highs: pd.Series, lows: pd.Series, window: int = 20 +) -> tuple[list[float], list[float]]: + """Find support/resistance using rolling min/max pivots.""" + n = len(highs) + if n < window * 2: + return [], [] + + h = highs.values + l = lows.values + close = float(highs.iloc[-1] + lows.iloc[-1]) / 2 + + levels: dict[float, int] = {} + for w in (window, window * 2): + if w >= n // 2: + continue + for i in range(w, n - w): + if l[i] == min(l[i - w : i + w + 1]): + lv = round(float(l[i]), 4) + levels[lv] = levels.get(lv, 0) + 1 + if h[i] == max(h[i - w : i + w + 1]): + lv = round(float(h[i]), 4) + levels[lv] = levels.get(lv, 0) + 1 + + if not levels: + return [], [] + + # Cluster nearby levels (within 0.3%) + sorted_levels = sorted(levels.items()) + clusters: list[list[tuple[float, int]]] = [[sorted_levels[0]]] + for price, hits in sorted_levels[1:]: + cluster_avg = np.mean([p for p, _ in clusters[-1]]) + if abs(price - cluster_avg) / cluster_avg < 0.003: + clusters[-1].append((price, hits)) + else: + clusters.append([(price, hits)]) + + merged = [] + for cluster in clusters: + total_hits = sum(h for _, h in cluster) + avg_price = np.average([p for p, _ in cluster], weights=[h for _, h in cluster]) + merged.append((round(float(avg_price), 4), total_hits)) + + supports = sorted([lv for lv, _ in merged if lv < close], reverse=True)[:3] + resistances = sorted([lv for lv, _ in merged if lv >= close])[:3] + return supports, resistances + + +# --------------------------------------------------------------------------- +# Data fetching & processing +# --------------------------------------------------------------------------- + +TIMEFRAMES = [ + {"label": "Intraday (1m)", "interval": "1m", "days": 1, "short_label": "1D"}, + {"label": "Semanal (15m)", "interval": "15m", "days": 7, "short_label": "1W"}, + {"label": "Mensual (1h)", "interval": "1h", "days": 30, "short_label": "1M"}, + {"label": "Trimestral (1d)", "interval": "1d", "days": 90, "short_label": "3M"}, +] + + +async def fetch_candles( + client, connector: str, pair: str, interval: str, days: int +) -> pd.DataFrame: + """Fetch candles and return as DataFrame.""" + result = await client.market_data.get_candles_last_days( + connector, pair, days=days, interval=interval + ) + # API returns a list directly, or a dict with "candles"/"data" key + if isinstance(result, list): + records = result + elif isinstance(result, dict): + records = result.get("data", result.get("candles", [])) + else: + records = [] + if not records: + return pd.DataFrame() + + df = pd.DataFrame(records) + for col in ("open", "high", "low", "close", "volume"): + df[col] = pd.to_numeric(df[col], errors="coerce") + + ts_col = df["timestamp"] + unit = "s" if ts_col.iloc[0] > 1e12 / 1000 else "ms" + df["datetime"] = pd.to_datetime(ts_col, unit=unit, utc=True) + df = df.sort_values("datetime").reset_index(drop=True) + return df + + +def compute_indicators(df: pd.DataFrame) -> dict: + """Compute all indicators for a timeframe.""" + close = df["close"] + sma20 = compute_sma(close, 20) + sma50 = compute_sma(close, 50) + ema9 = compute_ema(close, 9) + ema21 = compute_ema(close, 21) + rsi = compute_rsi(close, 14) + + current = float(close.iloc[-1]) + open_price = float(close.iloc[0]) + change_pct = (current - open_price) / open_price * 100 + + high = float(df["high"].max()) + low = float(df["low"].min()) + + supports, resistances = find_sr_levels(df["high"], df["low"]) + + # Trend direction + sma20_now = sma20.iloc[-1] + sma50_now = sma50.iloc[-1] + if pd.notna(sma20_now) and pd.notna(sma50_now): + if current > sma20_now > sma50_now: + trend = "ALCISTA" + elif current < sma20_now < sma50_now: + trend = "BAJISTA" + else: + trend = "LATERAL" + else: + trend = "N/A" + + return { + "sma20": sma20, + "sma50": sma50, + "ema9": ema9, + "ema21": ema21, + "rsi": rsi, + "current": current, + "change_pct": change_pct, + "high": high, + "low": low, + "supports": supports, + "resistances": resistances, + "trend": trend, + "rsi_now": float(rsi.iloc[-1]) if pd.notna(rsi.iloc[-1]) else 50, + "sma20_now": float(sma20_now) if pd.notna(sma20_now) else current, + "sma50_now": float(sma50_now) if pd.notna(sma50_now) else current, + } + + +# --------------------------------------------------------------------------- +# Chart +# --------------------------------------------------------------------------- + +COLORS = { + "bg_paper": "#0d1117", + "bg_plot": "#161b22", + "grid": "#21262d", + "text": "#c9d1d9", + "text_dim": "#8b949e", + "green": "#3fb950", + "red": "#f85149", + "sma20": "#f0883e", + "sma50": "#58a6ff", + "rsi_line": "#bc8cff", + "support": "#3fb950", + "resistance": "#f85149", +} + + +def build_chart(timeframe_data: list[dict], pair: str) -> io.BytesIO: + """Build a 4x2 dashboard: candlestick+SMA (left col), RSI (right col) per timeframe.""" + fig = make_subplots( + rows=4, + cols=2, + column_widths=[0.78, 0.22], + row_heights=[0.25, 0.25, 0.25, 0.25], + vertical_spacing=0.045, + horizontal_spacing=0.03, + subplot_titles=( + [ + item["label"] + for item in timeframe_data + for _ in range(2) # one title per subplot pair + ] + if False + else None + ), # We'll add custom titles + ) + + current_price = timeframe_data[0]["indicators"]["current"] if timeframe_data else 0 + + for i, item in enumerate(timeframe_data): + row = i + 1 + df = item["df"] + ind = item["indicators"] + tf = item["tf"] + dates = df["datetime"] + + # --- Candlestick --- + fig.add_trace( + go.Candlestick( + x=dates, + open=df["open"], + high=df["high"], + low=df["low"], + close=df["close"], + increasing_line_color=COLORS["green"], + decreasing_line_color=COLORS["red"], + increasing_fillcolor=COLORS["green"], + decreasing_fillcolor=COLORS["red"], + showlegend=False, + name=tf["short_label"], + ), + row=row, + col=1, + ) + + # SMA20 + sma20_valid = ind["sma20"].dropna() + if len(sma20_valid) > 0: + fig.add_trace( + go.Scatter( + x=dates.iloc[sma20_valid.index], + y=sma20_valid, + line=dict(color=COLORS["sma20"], width=1.2), + showlegend=(i == 0), + name="SMA 20", + ), + row=row, + col=1, + ) + + # SMA50 + sma50_valid = ind["sma50"].dropna() + if len(sma50_valid) > 0: + fig.add_trace( + go.Scatter( + x=dates.iloc[sma50_valid.index], + y=sma50_valid, + line=dict(color=COLORS["sma50"], width=1.2), + showlegend=(i == 0), + name="SMA 50", + ), + row=row, + col=1, + ) + + # Support lines + t0, t1 = dates.iloc[0], dates.iloc[-1] + for s in ind["supports"]: + fig.add_shape( + type="line", + x0=t0, + x1=t1, + y0=s, + y1=s, + line=dict(color=COLORS["support"], width=0.7, dash="dot"), + opacity=0.5, + row=row, + col=1, + ) + + # Resistance lines + for r in ind["resistances"]: + fig.add_shape( + type="line", + x0=t0, + x1=t1, + y0=r, + y1=r, + line=dict(color=COLORS["resistance"], width=0.7, dash="dot"), + opacity=0.5, + row=row, + col=1, + ) + + # --- RSI panel --- + rsi_valid = ind["rsi"].dropna() + if len(rsi_valid) > 0: + fig.add_trace( + go.Scatter( + x=dates.iloc[rsi_valid.index], + y=rsi_valid, + line=dict(color=COLORS["rsi_line"], width=1.5), + fill="tozeroy", + fillcolor="rgba(188,140,255,0.08)", + showlegend=False, + ), + row=row, + col=2, + ) + + # Overbought/oversold bands + fig.add_hline( + y=70, + line_dash="dot", + line_color=COLORS["red"], + opacity=0.4, + row=row, + col=2, + ) + fig.add_hline( + y=30, + line_dash="dot", + line_color=COLORS["green"], + opacity=0.4, + row=row, + col=2, + ) + + # --- Row label annotation --- + trend_color = COLORS["green"] if ind["change_pct"] >= 0 else COLORS["red"] + arrow = "+" if ind["change_pct"] >= 0 else "" + label_text = ( + f"{tf['short_label']} {tf['label'].split('(')[1].rstrip(')')}" + f" | {ind['current']:.3f}" + f" | {arrow}{ind['change_pct']:.2f}%" + f" | RSI: {ind['rsi_now']:.0f}" + f" | {ind['trend']}" + ) + fig.add_annotation( + text=label_text, + xref="paper", + yref="paper", + x=0, + y=1 - (i * 0.25) + 0.005, + showarrow=False, + font=dict(size=11, color=COLORS["text"]), + xanchor="left", + yanchor="bottom", + ) + + # --- Header --- + # Get quarterly change + q_change = ( + timeframe_data[3]["indicators"]["change_pct"] if len(timeframe_data) > 3 else 0 + ) + m_change = ( + timeframe_data[2]["indicators"]["change_pct"] if len(timeframe_data) > 2 else 0 + ) + w_change = ( + timeframe_data[1]["indicators"]["change_pct"] if len(timeframe_data) > 1 else 0 + ) + d_change = timeframe_data[0]["indicators"]["change_pct"] if timeframe_data else 0 + + title_text = ( + f"{pair} | {current_price:.3f}" + f" | D: {d_change:+.2f}% W: {w_change:+.2f}% M: {m_change:+.2f}% 3M: {q_change:+.2f}%" + ) + + # --- Layout --- + fig.update_layout( + title=dict(text=title_text, font=dict(color=COLORS["text"], size=14), x=0.5), + paper_bgcolor=COLORS["bg_paper"], + plot_bgcolor=COLORS["bg_plot"], + font=dict(color=COLORS["text_dim"], size=9), + legend=dict( + orientation="h", + yanchor="bottom", + y=1.05, + xanchor="center", + x=0.4, + font=dict(size=10, color=COLORS["text"]), + bgcolor="rgba(0,0,0,0)", + ), + margin=dict(l=55, r=30, t=100, b=30), + width=1300, + height=1400, + xaxis_rangeslider_visible=False, + xaxis3_rangeslider_visible=False, + xaxis5_rangeslider_visible=False, + xaxis7_rangeslider_visible=False, + ) + + # Style axes + for row in range(1, 5): + # Price axis + fig.update_yaxes( + gridcolor=COLORS["grid"], + zerolinecolor=COLORS["grid"], + tickformat=".3f", + row=row, + col=1, + ) + fig.update_xaxes( + gridcolor=COLORS["grid"], showticklabels=(row == 4), row=row, col=1 + ) + # RSI axis + fig.update_yaxes( + gridcolor=COLORS["grid"], + range=[0, 100], + tickvals=[30, 50, 70], + row=row, + col=2, + ) + fig.update_xaxes(gridcolor=COLORS["grid"], showticklabels=False, row=row, col=2) + + buf = io.BytesIO() + fig.write_image(buf, format="png", scale=2) + buf.seek(0) + return buf, fig + + +# --------------------------------------------------------------------------- +# Text summary +# --------------------------------------------------------------------------- + + +def format_summary(timeframe_data: list[dict], pair: str) -> str: + lines = [f"Dashboard {pair}"] + for item in timeframe_data: + ind = item["indicators"] + tf = item["tf"] + arrow = "+" if ind["change_pct"] >= 0 else "" + sup_str = ", ".join(f"{s:.3f}" for s in ind["supports"][:2]) or "N/A" + res_str = ", ".join(f"{r:.3f}" for r in ind["resistances"][:2]) or "N/A" + lines.append( + f"\n{tf['label']}:" + f"\n Precio: {ind['current']:.3f} ({arrow}{ind['change_pct']:.2f}%)" + f"\n Tendencia: {ind['trend']} | RSI: {ind['rsi_now']:.0f}" + f"\n SMA20: {ind['sma20_now']:.3f} | SMA50: {ind['sma50_now']:.3f}" + f"\n Soporte: {sup_str} | Resistencia: {res_str}" + f"\n Rango: {ind['low']:.3f} - {ind['high']:.3f}" + ) + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Telegram photo fallback +# --------------------------------------------------------------------------- + + +async def _send_photo_http(chat_id: int, buf: io.BytesIO, caption: str) -> None: + import os + + import httpx + + token = os.environ.get("TELEGRAM_BOT_TOKEN") or os.environ.get("TELEGRAM_TOKEN", "") + if not token: + return + + url = f"https://api.telegram.org/bot{token}/sendPhoto" + buf.seek(0) + async with httpx.AsyncClient(timeout=30) as http_client: + resp = await http_client.post( + url, + data={"chat_id": chat_id, "caption": caption}, + files={"photo": ("dashboard.png", buf, "image/png")}, + ) + if not resp.json().get("ok"): + logger.warning(f"sendPhoto failed: {resp.text}") + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: + chat_id = context._chat_id if hasattr(context, "_chat_id") else None + client = await get_client(chat_id, context=context) + if not client: + return "No server available" + + # Fetch all timeframes in parallel + fetch_tasks = [ + fetch_candles( + client, + config.connector_name, + config.trading_pair, + tf["interval"], + tf["days"], + ) + for tf in TIMEFRAMES + ] + results = await asyncio.gather(*fetch_tasks, return_exceptions=True) + + timeframe_data = [] + for tf, result in zip(TIMEFRAMES, results): + if isinstance(result, Exception): + logger.warning(f"Failed to fetch {tf['label']}: {result}") + continue + if result.empty or len(result) < 10: + logger.warning(f"Not enough data for {tf['label']}: {len(result)} candles") + continue + + indicators = compute_indicators(result) + timeframe_data.append( + { + "tf": tf, + "label": tf["label"], + "df": result, + "indicators": indicators, + } + ) + + if not timeframe_data: + return "No candle data available for any timeframe" + + # Generate chart + chart_bytes: bytes | None = None + plotly_fig = None + try: + buf, plotly_fig = build_chart(timeframe_data, config.trading_pair) + chart_bytes = buf.getvalue() + caption = f"{config.trading_pair} Multi-Timeframe Dashboard" + if chat_id: + buf.seek(0) + if context.bot: + await context.bot.send_photo( + chat_id=chat_id, photo=buf, caption=caption + ) + else: + await _send_photo_http(chat_id, buf, caption) + except Exception as e: + logger.warning(f"Chart generation/send failed: {e}") + + text = format_summary(timeframe_data, config.trading_pair) + + from condor.reports import ReportBuilder + + builder = ReportBuilder(f"Dashboard: {config.trading_pair}") + builder.source("routine", "brl_dashboard").tags(["dashboard", config.trading_pair]) + builder.markdown(text) + if plotly_fig is not None: + builder.plotly(plotly_fig) + await builder.save() + + if context.bot and chart_bytes: + return RoutineResult(text=text, chart_image=chart_bytes) + return text diff --git a/agents/brigado/routines/btc_brl_arbitrage.py b/agents/brigado/routines/btc_brl_arbitrage.py new file mode 100644 index 000000000..a51e3ccd1 --- /dev/null +++ b/agents/brigado/routines/btc_brl_arbitrage.py @@ -0,0 +1,608 @@ +"""BTC-BRL arbitrage: spread analysis + quoting strategy simulation.""" + +CATEGORY = "Market Data" + +import asyncio +import io +import logging + +import numpy as np +import pandas as pd +import plotly.graph_objects as go +from plotly.subplots import make_subplots +from pydantic import BaseModel, Field +from telegram.ext import ContextTypes + +from config_manager import get_client +from routines.base import RoutineResult + +logger = logging.getLogger(__name__) + +COLORS = { + "bg_paper": "#0d1117", + "bg_plot": "#161b22", + "grid": "#21262d", + "text": "#c9d1d9", + "text_dim": "#8b949e", + "usdt": "#f0b90b", + "brl": "#3fb950", + "spread": "#bc8cff", + "long": "#3fb950", + "short": "#f85149", + "pnl": "#58a6ff", + "fees": "#f78166", +} + +LAYOUT_BASE = dict( + paper_bgcolor=COLORS["bg_paper"], + plot_bgcolor=COLORS["bg_plot"], + font=dict(color=COLORS["text_dim"], size=10), + legend=dict( + orientation="h", + yanchor="bottom", + y=1.05, + xanchor="center", + x=0.5, + font=dict(size=10, color=COLORS["text"]), + bgcolor="rgba(0,0,0,0)", + ), + margin=dict(l=60, r=30, t=100, b=40), +) + + +class Config(BaseModel): + """BTC-BRL arbitrage: spread analysis + quoting strategy simulation.""" + + connector_name: str = Field(default="binance", description="Exchange connector") + lookback_seconds: int = Field( + default=3600, description="1s candles to fetch (3600=1h)" + ) + trade_size_usdt: float = Field( + default=100.0, description="Notional per trade in USDT" + ) + fee_bps: float = Field( + default=10.0, description="Round-trip fee in bps (maker+taker)" + ) + min_spread_bps: float = Field(default=2.0, description="Min spread to enter (bps)") + max_position_usdt: float = Field( + default=5000.0, description="Max accumulated position (USDT)" + ) + + +async def _fetch_candles( + client, connector: str, pair: str, interval: str, max_records: int +) -> pd.DataFrame: + result = await client.market_data.get_candles( + connector, pair, interval=interval, max_records=max_records + ) + if isinstance(result, list): + records = result + elif isinstance(result, dict): + records = result.get("data", result.get("candles", [])) + else: + records = [] + if not records: + return pd.DataFrame() + df = pd.DataFrame(records) + for col in ("open", "high", "low", "close", "volume", "quote_asset_volume"): + if col in df.columns: + df[col] = pd.to_numeric(df[col], errors="coerce") + ts = df["timestamp"] + unit = "s" if ts.iloc[0] > 1e12 / 1000 else "ms" + df["datetime"] = pd.to_datetime(ts, unit=unit, utc=True) + return df.sort_values("datetime").reset_index(drop=True) + + +def _style_axes(fig, rows): + for row in range(1, rows + 1): + fig.update_yaxes( + gridcolor=COLORS["grid"], zerolinecolor=COLORS["grid"], row=row, col=1 + ) + fig.update_xaxes(gridcolor=COLORS["grid"], row=row, col=1) + + +def _merge_spread( + price_usdt: pd.DataFrame, price_brl: pd.DataFrame, usdtbrl: pd.DataFrame +) -> pd.DataFrame: + """Merge 1s candles and compute spread %.""" + for df in (price_usdt, price_brl, usdtbrl): + df["ts"] = df["datetime"].dt.floor("s") + + merged = pd.merge( + price_usdt[["ts", "close"]].rename(columns={"close": "btc_usdt"}), + price_brl[["ts", "close"]].rename(columns={"close": "btc_brl"}), + on="ts", + how="inner", + ) + merged = pd.merge( + merged, + usdtbrl[["ts", "close"]].rename(columns={"close": "usdt_brl_rate"}), + on="ts", + how="inner", + ) + merged["btc_brl_usd"] = merged["btc_brl"] / merged["usdt_brl_rate"] + merged["spread_pct"] = ( + (merged["btc_brl_usd"] - merged["btc_usdt"]) / merged["btc_usdt"] + ) * 100 + merged["spread_bps"] = merged["spread_pct"] * 100 + return merged.sort_values("ts").reset_index(drop=True) + + +def simulate_quoting( + merged: pd.DataFrame, + trade_size_usdt: float, + fee_bps: float, + min_spread_bps: float, + max_position_usdt: float, +) -> pd.DataFrame: + """Simulate a one-sided quoting strategy that captures spread. + + Logic: + - When BRL premium (spread > min_spread): sell on BRL side (capture premium) + → accumulate short BRL / long USDT exposure + - When USDT premium (spread < -min_spread): sell on USDT side (capture premium) + → accumulate short USDT / long BRL exposure + - Position flips naturally as spread oscillates + - Fee deducted on each trade + """ + n = len(merged) + fee_pct = fee_bps / 10000 + + # Track state + position_usdt = np.zeros( + n + ) # net position in USDT terms (+ = long BRL, - = short BRL) + cum_captured = np.zeros(n) # cumulative spread captured (before fees) + cum_fees = np.zeros(n) # cumulative fees paid + cum_pnl = np.zeros(n) # net PnL (captured - fees) + trade_count = np.zeros(n, dtype=int) + actions = [""] * n + + pos = 0.0 + captured = 0.0 + fees = 0.0 + trades = 0 + + for i in range(n): + spread_bps = merged["spread_bps"].iloc[i] + abs_spread = abs(spread_bps) + + if abs_spread > min_spread_bps: + # Check position limits + if spread_bps > 0 and pos > -max_position_usdt: + # BRL premium → sell on BRL (go short BRL exposure) + trade_pnl = (abs_spread / 10000) * trade_size_usdt + trade_fee = fee_pct * trade_size_usdt + captured += trade_pnl + fees += trade_fee + pos -= trade_size_usdt + trades += 1 + actions[i] = "sell_brl" + elif spread_bps < 0 and pos < max_position_usdt: + # USDT premium → sell on USDT (go long BRL exposure) + trade_pnl = (abs_spread / 10000) * trade_size_usdt + trade_fee = fee_pct * trade_size_usdt + captured += trade_pnl + fees += trade_fee + pos += trade_size_usdt + trades += 1 + actions[i] = "sell_usdt" + + position_usdt[i] = pos + cum_captured[i] = captured + cum_fees[i] = fees + cum_pnl[i] = captured - fees + trade_count[i] = trades + + merged = merged.copy() + merged["position_usdt"] = position_usdt + merged["cum_captured"] = cum_captured + merged["cum_fees"] = cum_fees + merged["cum_pnl"] = cum_pnl + merged["trade_count"] = trade_count + merged["action"] = actions + + # Mark-to-market: unrealized PnL from position * current spread change + # Position was entered at various spreads; approximate with running avg + merged["unrealized"] = 0.0 + avg_entry_spread = 0.0 + total_pos = 0.0 + for i in range(n): + if actions[i]: + entry_spread = merged["spread_bps"].iloc[i] + trade_pos = ( + trade_size_usdt if actions[i] == "sell_usdt" else -trade_size_usdt + ) + if abs(total_pos + trade_pos) > 0.01: + avg_entry_spread = ( + avg_entry_spread * total_pos + entry_spread * trade_pos + ) / (total_pos + trade_pos) + total_pos += trade_pos + current_spread = merged["spread_bps"].iloc[i] + # If short BRL (pos < 0), we profit when spread decreases + # If long BRL (pos > 0), we profit when spread increases + spread_change = current_spread - avg_entry_spread + merged.loc[merged.index[i], "unrealized"] = (spread_change / 10000) * total_pos + + merged["total_pnl"] = merged["cum_pnl"] + merged["unrealized"] + + return merged + + +def build_spread_chart(merged: pd.DataFrame) -> go.Figure: + """Chart 1: Raw spread analysis.""" + avg_spread = merged["spread_pct"].mean() + curr_spread = merged["spread_pct"].iloc[-1] + std_spread = merged["spread_pct"].std() + max_spread = merged["spread_pct"].max() + min_spread = merged["spread_pct"].min() + current_btc = float(merged["btc_usdt"].iloc[-1]) + + fig = make_subplots( + rows=2, + cols=1, + row_heights=[0.55, 0.45], + vertical_spacing=0.08, + subplot_titles=[ + "BTC: USDT vs BRL→USD (1s)", + f"Spread % — avg {avg_spread:+.4f}%, std {std_spread:.4f}%", + ], + ) + + fig.add_trace( + go.Scatter( + x=merged["ts"], + y=merged["btc_usdt"], + line=dict(color=COLORS["usdt"], width=1.5), + name="BTC-USDT", + ), + row=1, + col=1, + ) + fig.add_trace( + go.Scatter( + x=merged["ts"], + y=merged["btc_brl_usd"], + line=dict(color=COLORS["brl"], width=1.5), + name="BTC-BRL (USD)", + ), + row=1, + col=1, + ) + + # Spread with color fill + fig.add_trace( + go.Scatter( + x=merged["ts"], + y=merged["spread_pct"], + line=dict(color=COLORS["spread"], width=1.2), + fill="tozeroy", + fillcolor="rgba(188,140,255,0.08)", + name="Spread %", + ), + row=2, + col=1, + ) + fig.add_hline(y=0, line_color=COLORS["grid"], row=2, col=1) + fig.add_hline( + y=avg_spread, + line_dash="dash", + line_color=COLORS["text_dim"], + opacity=0.6, + row=2, + col=1, + annotation_text=f"avg {avg_spread:+.4f}%", + annotation_font_color=COLORS["text_dim"], + ) + + fig.update_layout( + **LAYOUT_BASE, + title=dict( + text=( + f"BTC-BRL Spread | BTC: ${current_btc:,.0f}" + f" | Now: {curr_spread:+.4f}%" + f" | Range: [{min_spread:+.4f}%, {max_spread:+.4f}%]" + ), + font=dict(color=COLORS["text"], size=13), + x=0.5, + ), + width=1300, + height=700, + ) + fig.update_yaxes(title_text="USD", tickformat=",.0f", row=1, col=1) + fig.update_yaxes(title_text="%", tickformat=".4f", row=2, col=1) + for ann in fig.layout.annotations: + ann.font = dict(color=COLORS["text_dim"], size=11) + _style_axes(fig, 2) + return fig + + +def build_strategy_chart( + sim: pd.DataFrame, fee_bps: float, min_spread_bps: float, trade_size_usdt: float +) -> go.Figure: + """Chart 2: Strategy simulation results.""" + total_trades = int(sim["trade_count"].iloc[-1]) + final_pnl = sim["cum_pnl"].iloc[-1] + final_captured = sim["cum_captured"].iloc[-1] + final_fees = sim["cum_fees"].iloc[-1] + final_position = sim["position_usdt"].iloc[-1] + final_total = sim["total_pnl"].iloc[-1] + duration_min = (sim["ts"].iloc[-1] - sim["ts"].iloc[0]).total_seconds() / 60 + + fig = make_subplots( + rows=3, + cols=1, + row_heights=[0.4, 0.3, 0.3], + vertical_spacing=0.06, + subplot_titles=[ + f"Cumulative PnL — {total_trades} trades in {duration_min:.0f}min", + f"Net Position (max ${sim['position_usdt'].abs().max():,.0f})", + "Spread (bps) + Trade Signals", + ], + ) + + # Row 1: PnL breakdown + fig.add_trace( + go.Scatter( + x=sim["ts"], + y=sim["cum_captured"], + line=dict(color=COLORS["brl"], width=1.5), + name="Captured (gross)", + fill="tozeroy", + fillcolor="rgba(63,185,80,0.08)", + ), + row=1, + col=1, + ) + fig.add_trace( + go.Scatter( + x=sim["ts"], + y=sim["cum_fees"], + line=dict(color=COLORS["fees"], width=1, dash="dot"), + name="Fees paid", + ), + row=1, + col=1, + ) + fig.add_trace( + go.Scatter( + x=sim["ts"], + y=sim["cum_pnl"], + line=dict(color=COLORS["pnl"], width=2), + name="Net PnL (realized)", + ), + row=1, + col=1, + ) + fig.add_trace( + go.Scatter( + x=sim["ts"], + y=sim["total_pnl"], + line=dict(color=COLORS["spread"], width=1.5, dash="dash"), + name="Total PnL (realized + unrealized)", + ), + row=1, + col=1, + ) + + # Row 2: Position + pos_colors = [ + COLORS["long"] if v >= 0 else COLORS["short"] for v in sim["position_usdt"] + ] + fig.add_trace( + go.Scatter( + x=sim["ts"], + y=sim["position_usdt"], + line=dict(color=COLORS["pnl"], width=1.5), + fill="tozeroy", + fillcolor="rgba(88,166,255,0.08)", + name="Position (USDT)", + showlegend=True, + ), + row=2, + col=1, + ) + fig.add_hline(y=0, line_color=COLORS["grid"], row=2, col=1) + + # Row 3: Spread + signals + fig.add_trace( + go.Scatter( + x=sim["ts"], + y=sim["spread_bps"], + line=dict(color=COLORS["spread"], width=1), + name="Spread (bps)", + showlegend=True, + ), + row=3, + col=1, + ) + fig.add_hline( + y=min_spread_bps, + line_dash="dash", + line_color=COLORS["brl"], + opacity=0.5, + row=3, + col=1, + ) + fig.add_hline( + y=-min_spread_bps, + line_dash="dash", + line_color=COLORS["brl"], + opacity=0.5, + row=3, + col=1, + ) + fig.add_hline(y=0, line_color=COLORS["grid"], row=3, col=1) + + # Trade markers + sells_brl = sim[sim["action"] == "sell_brl"] + sells_usdt = sim[sim["action"] == "sell_usdt"] + if not sells_brl.empty: + fig.add_trace( + go.Scatter( + x=sells_brl["ts"], + y=sells_brl["spread_bps"], + mode="markers", + marker=dict(color=COLORS["short"], size=3, symbol="triangle-down"), + name="Sell BRL", + showlegend=True, + ), + row=3, + col=1, + ) + if not sells_usdt.empty: + fig.add_trace( + go.Scatter( + x=sells_usdt["ts"], + y=sells_usdt["spread_bps"], + mode="markers", + marker=dict(color=COLORS["long"], size=3, symbol="triangle-up"), + name="Sell USDT", + showlegend=True, + ), + row=3, + col=1, + ) + + fig.update_layout( + **LAYOUT_BASE, + title=dict( + text=( + f"Quoting Strategy Sim" + f" | ${trade_size_usdt:.0f}/trade" + f" | fee: {fee_bps:.0f}bps" + f" | min: {min_spread_bps:.0f}bps" + f" | PnL: ${final_pnl:+.2f}" + f" | Pos: ${final_position:+,.0f}" + ), + font=dict(color=COLORS["text"], size=12), + x=0.5, + ), + width=1300, + height=900, + ) + fig.update_yaxes(title_text="USDT", tickformat=",.2f", row=1, col=1) + fig.update_yaxes(title_text="USDT", tickformat=",.0f", row=2, col=1) + fig.update_yaxes(title_text="bps", row=3, col=1) + for ann in fig.layout.annotations: + ann.font = dict(color=COLORS["text_dim"], size=11) + _style_axes(fig, 3) + return fig + + +async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: + chat_id = context._chat_id + client = await get_client(chat_id, context=context) + if not client: + return RoutineResult(text="No server available") + + # Fetch 1s candles in parallel + btc_usdt, btc_brl, usdt_brl = await asyncio.gather( + _fetch_candles( + client, config.connector_name, "BTC-USDT", "1s", config.lookback_seconds + ), + _fetch_candles( + client, config.connector_name, "BTC-BRL", "1s", config.lookback_seconds + ), + _fetch_candles( + client, config.connector_name, "USDT-BRL", "1s", config.lookback_seconds + ), + ) + + for name, df in [ + ("BTC-USDT", btc_usdt), + ("BTC-BRL", btc_brl), + ("USDT-BRL", usdt_brl), + ]: + if df.empty: + return RoutineResult(text=f"No 1s data for {name}") + + # Compute spread + merged = _merge_spread(btc_usdt, btc_brl, usdt_brl) + if merged.empty: + return RoutineResult(text="No overlapping data between pairs") + + # Run strategy simulation + sim = simulate_quoting( + merged, + trade_size_usdt=config.trade_size_usdt, + fee_bps=config.fee_bps, + min_spread_bps=config.min_spread_bps, + max_position_usdt=config.max_position_usdt, + ) + + # Build charts + spread_fig = build_spread_chart(merged) + strategy_fig = build_strategy_chart( + sim, config.fee_bps, config.min_spread_bps, config.trade_size_usdt + ) + + # Summary stats + duration_min = (merged["ts"].iloc[-1] - merged["ts"].iloc[0]).total_seconds() / 60 + avg_spread = merged["spread_pct"].mean() + std_spread = merged["spread_pct"].std() + curr_spread = merged["spread_pct"].iloc[-1] + total_trades = int(sim["trade_count"].iloc[-1]) + final_captured = sim["cum_captured"].iloc[-1] + final_fees = sim["cum_fees"].iloc[-1] + final_pnl = sim["cum_pnl"].iloc[-1] + final_position = sim["position_usdt"].iloc[-1] + final_total = sim["total_pnl"].iloc[-1] + + # Spread distribution + above_thresh = (merged["spread_bps"].abs() > config.min_spread_bps).mean() * 100 + time_positive = (merged["spread_pct"] > 0).mean() * 100 + + # Annualize PnL + hours = duration_min / 60 + if hours > 0: + pnl_per_hour = final_pnl / hours + projected_daily = pnl_per_hour * 24 + projected_monthly = projected_daily * 30 + else: + pnl_per_hour = projected_daily = projected_monthly = 0 + + text = "\n".join( + [ + "BTC-BRL Arbitrage Analysis", + f"Window: {duration_min:.0f} min ({len(merged):,} ticks)", + "", + "Spread Stats:", + f" Current: {curr_spread:+.4f}% ({curr_spread*100:+.2f} bps)", + f" Average: {avg_spread:+.4f}%", + f" Std Dev: {std_spread:.4f}%", + f" Time |spread| > {config.min_spread_bps:.0f}bps: {above_thresh:.1f}%", + f" Time BRL premium: {time_positive:.1f}%", + "", + f"Strategy Sim (${config.trade_size_usdt:.0f}/trade, {config.fee_bps:.0f}bps fee, >{config.min_spread_bps:.0f}bps):", + f" Trades: {total_trades}", + f" Gross captured: ${final_captured:.2f}", + f" Fees paid: ${final_fees:.2f}", + f" Net realized PnL: ${final_pnl:+.2f}", + f" Current position: ${final_position:+,.0f}", + f" Total PnL (incl. unrealized): ${final_total:+.2f}", + "", + "Projections (if conditions persist):", + f" Per hour: ${pnl_per_hour:+.2f}", + f" Per day: ${projected_daily:+.2f}", + f" Per month: ${projected_monthly:+.2f}", + ] + ) + + # Export primary chart + img_bytes = spread_fig.to_image(format="png", width=1300, height=700, scale=2) + chart = io.BytesIO(img_bytes) + + # Report + from condor.reports import ReportBuilder + + builder = ReportBuilder("BTC-BRL Arbitrage") + builder.source("routine", "btc_brl_arbitrage").tags( + ["btc", "brl", "arbitrage", "strategy"] + ) + builder.markdown(text) + builder.plotly(spread_fig) + builder.plotly(strategy_fig) + await builder.save() + + return RoutineResult(text=text, chart_image=chart) diff --git a/agents/brigado/routines/config_analyzer.py b/agents/brigado/routines/config_analyzer.py new file mode 100644 index 000000000..ca811cd0c --- /dev/null +++ b/agents/brigado/routines/config_analyzer.py @@ -0,0 +1,772 @@ +"""PMM Config Analyzer — Compare controller configs side by side, highlight differences.""" + +CATEGORY = "Bot Analysis" + +import asyncio +import logging +import re +from collections import defaultdict + +import plotly.graph_objects as go +from pydantic import BaseModel, Field +from telegram.ext import ContextTypes + +from config_manager import get_config_manager + +logger = logging.getLogger(__name__) + +_DARK_BG = "#1a1a2e" +_DARK_PAPER = "#16213e" +_GREEN = "#00d4aa" +_RED = "#ff6b6b" +_BLUE = "#58a6ff" +_YELLOW = "#ffdd57" +_COLORS = [ + "#58a6ff", + "#00d4aa", + "#ff6b6b", + "#ffdd57", + "#c084fc", + "#fb923c", + "#38bdf8", + "#f472b6", +] + +# Parameters to always skip in comparison (noise) +SKIP_PARAMS = {"id", "controller_type", "controller_name", "manual_kill_switch"} + +# Parameters to highlight when different (key tuning params) +KEY_PARAMS = { + "buy_spreads", + "sell_spreads", + "buy_amounts_pct", + "sell_amounts_pct", + "total_amount_quote", + "stop_loss", + "take_profit", + "time_limit", + "executor_refresh_time", + "cooldown_time", + "leverage", + "dca_spreads", + "dca_amounts", + "trailing_stop", + "executor_activation_bounds", + "top_executor_refresh_time", + "position_rebalance_threshold_pct", +} + + +class Config(BaseModel): + """Analyze and compare PMM controller configs — saved vs running bots.""" + + server_name: str = Field( + default="brigado_2", + description="Server to connect to", + ) + controller_type: str = Field( + default="", + description="Filter by controller type (e.g. 'market_making'). Empty = all types.", + ) + controller_filter: str = Field( + default="pmm_mister", + description="Filter by config name or controller_name containing this string. Empty = all.", + ) + pair_filter: str = Field( + default="", + description="Filter by trading pair (e.g. 'BRL', 'BTC-BRL'). Empty = all.", + ) + running_only: bool = Field( + default=False, + description="Show only configs that are currently running in active bots.", + ) + show_common: bool = Field( + default=False, + description="Include parameters that are the same across all configs (verbose mode).", + ) + + +def _short_name(config_name: str) -> str: + """Shorten config name for display.""" + name = config_name + name = re.sub(r"^mm_", "", name) + name = re.sub(r"pmm_?mister_?", "pm_", name) + name = re.sub(r"^(binance|okx|bybit|kucoin)(_perpetual)?_", "", name) + name = re.sub(r"config[_]?(\d+)", r"c\1", name) + while "__" in name: + name = name.replace("__", "_") + return name.strip("_") or config_name[:25] + + +def _fmt_value(v) -> str: + """Format a config value for display.""" + if v is None: + return "—" + if isinstance(v, bool): + return "✓" if v else "✗" + if isinstance(v, float): + if v < 0.001: + return f"{v:.5f}" + if v < 0.01: + return f"{v:.4f}" + return f"{v:.3f}" if v < 10 else f"{v:.1f}" + if isinstance(v, list): + return ", ".join(_fmt_value(x) for x in v) + if isinstance(v, dict): + parts = [f"{k}={_fmt_value(vv)}" for k, vv in v.items()] + return " | ".join(parts) + return str(v) + + +def _normalize_value(v): + """Normalize values for comparison.""" + if v is None: + return None + if isinstance(v, str): + try: + return float(v) + except ValueError: + return v.strip().lower() + if isinstance(v, list): + return [_normalize_value(x) for x in v] + if isinstance(v, dict): + return {k: _normalize_value(vv) for k, vv in v.items()} + return v + + +async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: + # Connect to server + client = await get_config_manager().get_client(config.server_name) + if not client: + return f"Could not connect to server '{config.server_name}'" + + # Fetch all data in parallel + all_configs_list, bots_data = await asyncio.gather( + client.controllers.list_controller_configs(), + _get_bots_data(client), + ) + + # --- Saved configs --- + saved_configs = {} + filt = config.controller_filter.lower() if config.controller_filter else "" + for cfg_meta in all_configs_list or []: + cname = cfg_meta.get("id") or cfg_meta.get("config_name", "") + ctype = cfg_meta.get("controller_type", "") + ctrl_name = cfg_meta.get("controller_name", "") + + if ( + config.controller_type + and config.controller_type.lower() not in ctype.lower() + ): + continue + if filt and filt not in cname.lower() and filt not in ctrl_name.lower(): + continue + + try: + cfg_data = await client.controllers.get_controller_config(cname) + # Also check controller_name inside the full config data + if filt and filt not in cname.lower(): + inner_ctrl = cfg_data.get("controller_name", "") + if filt not in inner_ctrl.lower(): + continue + if config.pair_filter: + pair = cfg_data.get("trading_pair", "") + if config.pair_filter.upper() not in pair.upper(): + continue + saved_configs[cname] = cfg_data + except Exception as e: + logger.warning(f"Failed to fetch config {cname}: {e}") + + # --- Running bot configs --- + running_config_ids = set() + running_configs = {} + for bot_name in bots_data or {}: + try: + bot_cfgs = await client.controllers.get_bot_controller_configs(bot_name) + if isinstance(bot_cfgs, list): + for cfg in bot_cfgs: + cid = cfg.get("id") or cfg.get("controller_id", "") + ctype = cfg.get("controller_type", "") + if ( + config.controller_type + and config.controller_type.lower() not in ctype.lower() + ): + continue + ctrl_name_inner = cfg.get("controller_name", "") + if ( + filt + and filt not in cid.lower() + and filt not in ctrl_name_inner.lower() + ): + continue + if config.pair_filter: + pair = cfg.get("trading_pair", "") + if config.pair_filter.upper() not in pair.upper(): + continue + running_configs[cid] = cfg + running_config_ids.add(cid) + except Exception as e: + logger.warning(f"Failed to fetch bot configs for {bot_name}: {e}") + + # Merge all configs with status prefix + all_configs = {} + for name, cfg in saved_configs.items(): + prefix = "🟢" if name in running_config_ids else "🔵" + all_configs[f"{prefix} {name}"] = cfg + for cid, cfg in running_configs.items(): + if cid not in saved_configs: + all_configs[f"🟢 {cid}"] = cfg + + # Filter to running only if requested + if config.running_only: + all_configs = { + k: v for k, v in all_configs.items() if k.startswith("\U0001f7e2") + } + + if not all_configs: + filters = [] + if config.controller_type: + filters.append(f"type={config.controller_type}") + if config.controller_filter: + filters.append(f"filter={config.controller_filter}") + if config.pair_filter: + filters.append(f"pair={config.pair_filter}") + return f"No configs found matching: {', '.join(filters) if filters else 'any'}" + + # --- Build comparison --- + all_params = set() + for cfg in all_configs.values(): + all_params.update(cfg.keys()) + all_params -= SKIP_PARAMS + + # Sort: key params first, then alphabetical + sorted_params = sorted(all_params, key=lambda p: (0 if p in KEY_PARAMS else 1, p)) + + # Find which params differ + diff_params = set() + for param in sorted_params: + values = [_normalize_value(cfg.get(param)) for cfg in all_configs.values()] + non_none = [v for v in values if v is not None] + if len(set(str(v) for v in non_none)) > 1: + diff_params.add(param) + + # Group configs by trading pair + pair_groups = defaultdict(dict) + for label, cfg in all_configs.items(): + pair = cfg.get("trading_pair", "unknown") + pair_groups[pair][label] = cfg + + num_running = sum(1 for l in all_configs if l.startswith("🟢")) + + # --- Build HTML report --- + from condor.reports import ReportBuilder + + builder = ReportBuilder("PMM Config Analyzer") + builder.source("routine", "config_analyzer").tags(["configs", "pmm", "analysis"]) + builder.manual_order() + + # KPIs + builder.kpi("Total Configs", str(len(all_configs))) + builder.kpi("Running", str(num_running), trend="up" if num_running else "neutral") + builder.kpi("Saved Only", str(len(all_configs) - num_running)) + builder.kpi( + "Diffs Found", + str(len(diff_params)), + delta=f"of {len(sorted_params)} params", + ) + builder.kpi("Trading Pairs", str(len(pair_groups))) + + # --- Per-pair comparison tables --- + for pair, pair_cfgs in sorted(pair_groups.items(), key=lambda x: -len(x[1])): + n_running = sum(1 for l in pair_cfgs if l.startswith("🟢")) + + labels = list(pair_cfgs.keys()) + shorts = [_short_name(l.replace("🟢 ", "").replace("🔵 ", "")) for l in labels] + + # Find pair-level diffs + pair_diff_params = set() + for param in sorted_params: + values = [_normalize_value(pair_cfgs[l].get(param)) for l in labels] + non_none = [v for v in values if v is not None] + if len(non_none) > 0 and len(set(str(v) for v in non_none)) > 1: + pair_diff_params.add(param) + + heading_md = f"## {pair} ({len(pair_cfgs)} configs, {n_running} running)" + + if len(labels) == 1: + # Single config — show full params + label = labels[0] + cfg = pair_cfgs[label] + rows = [] + for param in sorted_params: + val = cfg.get(param) + if val is not None: + is_key = "⚡ " if param in KEY_PARAMS else "" + rows.append( + { + "Parameter": f"{is_key}{param}", + "Value": _fmt_value(val), + } + ) + builder.markdown(f"{heading_md}\n\n**{label}** — single config") + if rows: + builder.table(rows) + else: + # Multiple configs — comparison view + rows = [] + for param in sorted_params: + values = [pair_cfgs[l].get(param) for l in labels] + non_none = [v for v in values if v is not None] + if not non_none: + continue + + is_diff = param in pair_diff_params + if not config.show_common and not is_diff: + continue + + prefix = ( + "🔴 " + if is_diff and param in KEY_PARAMS + else "🟡 " if is_diff else "" + ) + row = {"Parameter": f"{prefix}{param}"} + for short, val in zip(shorts, values): + row[short] = _fmt_value(val) + rows.append(row) + + note = ( + f"Showing {len(rows)} differing params. 🔴 = key, 🟡 = other." + if not config.show_common + else "" + ) + if not rows and not config.show_common: + builder.markdown( + f"{heading_md}\n\nAll parameters identical across these configs." + ) + else: + builder.markdown(f"{heading_md}\n\n{note}" if note else heading_md) + if rows: + builder.table(rows) + + # --- Cross-pair key differences summary --- + if len(pair_groups) > 1 and diff_params: + cross_sections = [] + for param in sorted_params: + if param not in diff_params or param not in KEY_PARAMS: + continue + value_groups = defaultdict(list) + for label, cfg in all_configs.items(): + val = _fmt_value(cfg.get(param)) + short = _short_name(label.replace("🟢 ", "").replace("🔵 ", "")) + value_groups[val].append(short) + if len(value_groups) <= 1: + continue + rows = [] + for val, names in sorted(value_groups.items(), key=lambda x: -len(x[1])): + rows.append( + { + "Value": val, + "Count": str(len(names)), + "Configs": ", ".join(names[:8]) + + (f" (+{len(names)-8})" if len(names) > 8 else ""), + } + ) + cross_sections.append((param, rows)) + + if cross_sections: + builder.markdown( + "## Cross-Pair Key Differences\n\nKey tuning parameters grouped by distinct values across all configs." + ) + for param, rows in cross_sections: + builder.markdown(f"**`{param}`** — {len(rows)} distinct values:") + builder.table(rows) + + # --- Charts --- + if len(all_configs) > 1: + fig_pc = _build_parallel_coords(all_configs, running_config_ids) + if fig_pc: + builder.plotly(fig_pc) + + fig = _build_spread_chart(all_configs) + if fig: + builder.plotly(fig) + + fig2 = _build_capital_chart(all_configs) + if fig2: + builder.plotly(fig2) + + await builder.save() + + # --- Concise plain text summary --- + lines = [f"**PMM Config Analyzer** — {config.server_name}", ""] + lines.append( + f"Found **{len(all_configs)}** configs ({num_running} running, {len(all_configs) - num_running} saved-only)" + ) + lines.append(f"**{len(diff_params)}** parameters differ across configs") + lines.append( + f"**{len(pair_groups)}** pairs: {', '.join(sorted(pair_groups.keys()))}" + ) + lines.append("") + + for pair, pair_cfgs in sorted(pair_groups.items(), key=lambda x: -len(x[1])): + n_r = sum(1 for l in pair_cfgs if l.startswith("🟢")) + lines.append(f"- **{pair}**: {len(pair_cfgs)} configs ({n_r} running)") + + lines.append("") + lines.append("Full comparison report available in the dashboard.") + + return "\n".join(lines) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +async def _get_bots_data(client) -> dict: + try: + data = await client.bot_orchestration.get_active_bots_status() + if isinstance(data, dict) and "data" in data: + return data["data"] + except Exception: + pass + return {} + + +def _build_parallel_coords(all_configs: dict, running_ids: set) -> go.Figure | None: + """Parallel coordinates plot (HiPlot-style) for numeric config parameters.""" + # Extract numeric axes from key params + NUMERIC_AXES = [ + ("total_amount_quote", "Capital", None), + ( + "buy_spreads", + "Buy Spread L1 %", + lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, + ), + ( + "sell_spreads", + "Sell Spread L1 %", + lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, + ), + ( + "buy_spreads", + "Buy Spread L2 %", + lambda v: ( + (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None + ), + ), + ( + "sell_spreads", + "Sell Spread L2 %", + lambda v: ( + (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None + ), + ), + ("executor_refresh_time", "Refresh Time", None), + ("cooldown_time", "Cooldown", None), + ("leverage", "Leverage", None), + ("stop_loss", "Stop Loss", None), + ("take_profit", "Take Profit", None), + ("time_limit", "Time Limit", None), + ("top_executor_refresh_time", "Top Refresh", None), + ( + "dca_spreads", + "DCA Spread L1", + lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, + ), + ( + "dca_amounts", + "DCA Amt L1", + lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, + ), + ( + "trailing_stop", + "Trail Stop", + lambda v: ( + (v.get("activation_price_delta") if isinstance(v, dict) else v) + if v + else None + ), + ), + ( + "executor_activation_bounds", + "Activation Bounds", + lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, + ), + ] + + # Collect data rows + rows = [] + labels = [] + for label, cfg in all_configs.items(): + clean_label = label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", "") + is_running = clean_label in running_ids or label.startswith("\U0001f7e2") + pair = cfg.get("trading_pair", "unknown") + row = {"_running": 1 if is_running else 0, "_pair": pair} + for param, axis_name, extractor in NUMERIC_AXES: + raw = cfg.get(param) + if extractor: + val = extractor(raw) + else: + val = raw + try: + row[axis_name] = float(val) if val is not None else None + except (TypeError, ValueError): + row[axis_name] = None + rows.append(row) + labels.append(_short_name(clean_label)) + + if len(rows) < 2: + return None + + # Filter axes that have at least 2 distinct non-None values + valid_axes = [] + for _, axis_name, _ in NUMERIC_AXES: + vals = [r[axis_name] for r in rows if r[axis_name] is not None] + if len(vals) >= 2 and len(set(vals)) > 1: + valid_axes.append(axis_name) + + if len(valid_axes) < 2: + return None + + # Build color by pair + unique_pairs = sorted(set(r["_pair"] for r in rows)) + pair_to_idx = {p: i for i, p in enumerate(unique_pairs)} + color_values = [pair_to_idx[r["_pair"]] for r in rows] + + # Build colorscale from our palette + n_pairs = len(unique_pairs) + if n_pairs == 1: + colorscale = [[0, _BLUE], [1, _BLUE]] + else: + colorscale = [] + for i in range(n_pairs): + pos = i / (n_pairs - 1) + colorscale.append([pos, _COLORS[i % len(_COLORS)]]) + + # Fill None values with axis min (so lines don't break) + for axis_name in valid_axes: + vals = [r[axis_name] for r in rows if r[axis_name] is not None] + fill = min(vals) if vals else 0 + for r in rows: + if r[axis_name] is None: + r[axis_name] = fill + + # Build dimensions + dimensions = [] + for axis_name in valid_axes: + vals = [r[axis_name] for r in rows] + # Multiply spread values by 100 for readability + if "%" in axis_name: + vals = [v * 100 for v in vals] + dimensions.append( + dict( + label=axis_name, + values=vals, + range=( + [min(vals), max(vals)] + if min(vals) != max(vals) + else [min(vals) - 1, max(vals) + 1] + ), + ) + ) + + fig = go.Figure( + data=go.Parcoords( + line=dict( + color=color_values, + colorscale=colorscale, + showscale=False, + ), + dimensions=dimensions, + customdata=labels, + ) + ) + + # Add pair legend via invisible scatter traces + for i, pair in enumerate(unique_pairs): + fig.add_trace( + go.Scatter( + x=[None], + y=[None], + mode="markers", + marker=dict(size=10, color=_COLORS[i % len(_COLORS)]), + name=pair, + showlegend=True, + ) + ) + + fig.update_layout( + title=dict( + text=f"Config Parameter Space ({len(rows)} configs, {len(valid_axes)} axes)", + font=dict(color="#e6edf3", size=14), + x=0.5, + ), + paper_bgcolor=_DARK_PAPER, + plot_bgcolor=_DARK_BG, + font=dict(color="#ccc", size=11), + margin=dict(l=80, r=80, t=80, b=40), + width=1400, + height=600, + legend=dict( + orientation="h", y=-0.05, x=0.5, xanchor="center", font=dict(size=12) + ), + ) + return fig + + +def _build_spread_chart(all_configs: dict) -> go.Figure | None: + """Grouped bar chart comparing buy/sell spreads across configs.""" + + def _to_list(v) -> list: + if v is None: + return [] + if isinstance(v, (int, float)): + return [float(v)] + if isinstance(v, list): + return [float(x) for x in v] + return [] + + configs_with_spreads = [] + for label, cfg in all_configs.items(): + buy = _to_list(cfg.get("buy_spreads")) + sell = _to_list(cfg.get("sell_spreads")) + if buy or sell: + short = _short_name(label.replace("🟢 ", "").replace("🔵 ", "")) + pair = cfg.get("trading_pair", "") + configs_with_spreads.append( + { + "name": short, + "pair": pair, + "buy_l1": buy[0] * 100 if len(buy) >= 1 else 0, + "sell_l1": sell[0] * 100 if len(sell) >= 1 else 0, + "buy_l2": buy[1] * 100 if len(buy) >= 2 else 0, + "sell_l2": sell[1] * 100 if len(sell) >= 2 else 0, + } + ) + + if not configs_with_spreads: + return None + + configs_with_spreads.sort(key=lambda c: (c["pair"], c["name"])) + names = [c["name"] for c in configs_with_spreads] + + fig = go.Figure() + fig.add_trace( + go.Bar( + x=names, + y=[c["buy_l1"] for c in configs_with_spreads], + name="Buy L1", + marker_color=_GREEN, + opacity=0.9, + ) + ) + fig.add_trace( + go.Bar( + x=names, + y=[c["sell_l1"] for c in configs_with_spreads], + name="Sell L1", + marker_color=_RED, + opacity=0.9, + ) + ) + + has_l2 = any(c["buy_l2"] > 0 or c["sell_l2"] > 0 for c in configs_with_spreads) + if has_l2: + fig.add_trace( + go.Bar( + x=names, + y=[c["buy_l2"] for c in configs_with_spreads], + name="Buy L2", + marker_color=_GREEN, + opacity=0.5, + ) + ) + fig.add_trace( + go.Bar( + x=names, + y=[c["sell_l2"] for c in configs_with_spreads], + name="Sell L2", + marker_color=_RED, + opacity=0.5, + ) + ) + + fig.update_layout( + title=dict( + text="Spread Levels by Config (%)", + font=dict(color="#e6edf3", size=14), + x=0.5, + ), + paper_bgcolor=_DARK_PAPER, + plot_bgcolor=_DARK_BG, + font=dict(color="#ccc", size=10), + barmode="group", + yaxis=dict(gridcolor="#2a2a4a", title="Spread %"), + xaxis=dict(gridcolor="#2a2a4a", tickangle=-45), + legend=dict(orientation="h", y=1.1, x=0.5, xanchor="center"), + margin=dict(l=60, r=40, t=60, b=120), + width=1200, + height=500, + ) + return fig + + +def _build_capital_chart(all_configs: dict) -> go.Figure | None: + """Horizontal bar chart: capital allocation per config, colored by pair.""" + data = [] + for label, cfg in all_configs.items(): + amt = float(cfg.get("total_amount_quote", 0) or 0) + if amt <= 0: + continue + short = _short_name(label.replace("🟢 ", "").replace("🔵 ", "")) + pair = cfg.get("trading_pair", "") + is_running = label.startswith("🟢") + data.append({"name": short, "pair": pair, "amount": amt, "running": is_running}) + + if not data: + return None + + data.sort(key=lambda d: (d["pair"], d["amount"])) + + unique_pairs = list(dict.fromkeys(d["pair"] for d in data)) + pair_colors = {p: _COLORS[i % len(_COLORS)] for i, p in enumerate(unique_pairs)} + + fig = go.Figure() + for pair in unique_pairs: + pair_data = [d for d in data if d["pair"] == pair] + fig.add_trace( + go.Bar( + y=[d["name"] for d in pair_data], + x=[d["amount"] for d in pair_data], + orientation="h", + name=pair, + marker_color=pair_colors[pair], + opacity=0.85, + text=[ + f"{'🟢' if d['running'] else ''} {d['amount']:,.0f}" + for d in pair_data + ], + textposition="auto", + ) + ) + + height = max(400, len(data) * 22 + 100) + fig.update_layout( + title=dict( + text="Capital Allocation by Config (quote currency)", + font=dict(color="#e6edf3", size=14), + x=0.5, + ), + paper_bgcolor=_DARK_PAPER, + plot_bgcolor=_DARK_BG, + font=dict(color="#ccc", size=10), + barmode="stack", + xaxis=dict(gridcolor="#2a2a4a", title="Amount (quote)"), + yaxis=dict(gridcolor="#2a2a4a"), + legend=dict(orientation="h", y=1.05, x=0.5, xanchor="center"), + margin=dict(l=150, r=40, t=60, b=50), + width=1200, + height=height, + ) + return fig diff --git a/agents/brigado/routines/spot_markets.py b/agents/brigado/routines/spot_markets.py new file mode 100644 index 000000000..a3795b995 --- /dev/null +++ b/agents/brigado/routines/spot_markets.py @@ -0,0 +1,458 @@ +"""Spot market discovery — top markets by volume across quote currencies with rebate focus.""" + +CATEGORY = "Market Data" + +import asyncio +import logging +from typing import Any + +import aiohttp +import pandas as pd +import plotly.graph_objects as go +from plotly.subplots import make_subplots +from pydantic import BaseModel, Field +from telegram.ext import ContextTypes + +from config_manager import get_client +from routines.base import RoutineResult + +logger = logging.getLogger(__name__) + +BINANCE_SPOT_TICKER = "https://api.binance.com/api/v3/ticker/24hr" + +# Quote currencies we care about +REBATE_QUOTES = { + "BRL", + "ARS", + "MXN", + "EUR", + "PLN", + "JPY", + "TRY", + "RON", + "UAH", + "ZAR", + "NGN", + "COP", +} +STABLE_QUOTES = {"USDT", "USDC", "BUSD", "FDUSD", "TUSD"} + +# Stablecoins / wrapped tokens to exclude as base +EXCLUDED_BASES = {"USDT", "USDC", "BUSD", "TUSD", "FDUSD", "DAI", "BFUSD"} + +COLORS = { + "bg_paper": "#0d1117", + "bg_plot": "#161b22", + "grid": "#21262d", + "text": "#c9d1d9", + "text_dim": "#8b949e", + "green": "#3fb950", + "red": "#f85149", + "blue": "#58a6ff", + "yellow": "#f0b90b", + "purple": "#bc8cff", + "orange": "#f78166", +} + +QUOTE_COLORS = { + "BRL": "#3fb950", + "USDT": "#e6e6e6", + "FDUSD": "#58a6ff", + "USDC": "#2775ca", + "ARS": "#75b5f0", + "MXN": "#f0b90b", + "EUR": "#f78166", + "PLN": "#bc8cff", + "JPY": "#ff7b72", + "TRY": "#d2a8ff", + "RON": "#7ee787", + "UAH": "#79c0ff", + "ZAR": "#ffa657", + "NGN": "#56d364", + "COP": "#e3b341", +} + + +class Config(BaseModel): + """Spot market discovery — top pairs by volume, FIAT rebate markets, and USDT leaders.""" + + top_n: int = Field(default=50, description="Top N markets per section") + min_volume_usd: float = Field( + default=100_000, description="Min 24h volume (USD) for FIAT pairs" + ) + min_volume_usdt: float = Field( + default=5_000_000, description="Min 24h volume (USD) for USDT pairs" + ) + focus_quote: str = Field( + default="BRL", description="Primary FIAT quote to highlight" + ) + + +def _parse_quote(symbol: str, known_quotes: set[str]) -> tuple[str, str] | None: + """Parse a Binance symbol into (base, quote) using known quote list.""" + for q in sorted(known_quotes, key=len, reverse=True): + if symbol.endswith(q): + base = symbol[: -len(q)] + if base: + return base, q + return None + + +async def fetch_spot_tickers() -> list[dict]: + """Fetch all 24h tickers from Binance spot.""" + async with aiohttp.ClientSession() as session: + async with session.get(BINANCE_SPOT_TICKER) as resp: + resp.raise_for_status() + return await resp.json() + + +def categorize_tickers(tickers: list[dict]) -> dict[str, list[dict]]: + """Parse and categorize tickers by quote currency.""" + all_quotes = REBATE_QUOTES | STABLE_QUOTES | {"BTC", "ETH", "BNB"} + by_quote: dict[str, list[dict]] = {} + + for t in tickers: + symbol = t.get("symbol", "") + parsed = _parse_quote(symbol, all_quotes) + if not parsed: + continue + base, quote = parsed + if base in EXCLUDED_BASES: + continue + + quote_vol = float(t.get("quoteVolume", 0)) + if quote_vol <= 0: + continue + + entry = { + "symbol": symbol, + "base": base, + "quote": quote, + "trading_pair": f"{base}-{quote}", + "price": float(t.get("lastPrice", 0)), + "price_change_pct": float(t.get("priceChangePercent", 0)), + "volume_24h": float(t.get("volume", 0)), + "quote_volume_24h": quote_vol, + "trades_24h": int(t.get("count", 0)), + } + by_quote.setdefault(quote, []).append(entry) + + # Sort each group by quote volume + for q in by_quote: + by_quote[q].sort(key=lambda x: x["quote_volume_24h"], reverse=True) + + return by_quote + + +async def _fetch_usdt_rates(tickers: list[dict]) -> dict[str, float]: + """Extract USDT rates for fiat currencies from ticker data.""" + rates = {"USDT": 1.0, "USDC": 1.0, "FDUSD": 1.0, "BUSD": 1.0, "TUSD": 1.0} + # BTC-USDT price for cross-conversion + btc_usdt = 0.0 + for t in tickers: + sym = t.get("symbol", "") + price = float(t.get("lastPrice", 0)) + if sym == "BTCUSDT": + btc_usdt = price + # Direct USDT/FIAT pairs (e.g., USDTBRL → 1 USDT = X BRL) + if sym.startswith("USDT") and price > 0: + fiat = sym[4:] + if fiat in REBATE_QUOTES: + rates[fiat] = 1.0 / price # BRL per USDT → USDT per BRL + # Also check FIAT/USDT pairs (unlikely on Binance but just in case) + for t in tickers: + sym = t.get("symbol", "") + price = float(t.get("lastPrice", 0)) + if sym.endswith("USDT") and price > 0: + base = sym[:-4] + if base in REBATE_QUOTES and base not in rates: + rates[base] = price + return rates + + +def format_volume(v: float) -> str: + if v >= 1_000_000_000: + return f"${v / 1_000_000_000:.1f}B" + if v >= 1_000_000: + return f"${v / 1_000_000:.1f}M" + if v >= 1_000: + return f"${v / 1_000:.0f}K" + return f"${v:.0f}" + + +def _build_report( + by_quote: dict[str, list[dict]], + usdt_rates: dict[str, float], + config: Config, +) -> tuple[str, go.Figure, list[dict]]: + """Build text summary, chart, and table data.""" + + # --- Section 1: Focus FIAT quote (BRL by default) --- + focus = config.focus_quote.upper() + focus_pairs = by_quote.get(focus, []) + focus_rate = usdt_rates.get(focus, 0) + + # --- Section 2: All FIAT markets summary --- + fiat_summary = [] + for q in sorted(REBATE_QUOTES): + pairs = by_quote.get(q, []) + if not pairs: + continue + rate = usdt_rates.get(q, 0) + total_vol_usdt = ( + sum(p["quote_volume_24h"] * rate for p in pairs) if rate > 0 else 0 + ) + fiat_summary.append( + { + "quote": q, + "pair_count": len(pairs), + "total_volume_usdt": total_vol_usdt, + "top_pair": pairs[0]["trading_pair"] if pairs else "-", + "top_volume": ( + pairs[0]["quote_volume_24h"] * rate if pairs and rate > 0 else 0 + ), + } + ) + fiat_summary.sort(key=lambda x: x["total_volume_usdt"], reverse=True) + + # --- Section 3: Top USDT pairs --- + usdt_pairs = [ + p + for p in by_quote.get("USDT", []) + if p["quote_volume_24h"] >= config.min_volume_usdt + ] + usdt_top = usdt_pairs[: config.top_n] + + # --- Build text --- + lines = [] + + # FIAT overview + lines.append("FIAT MARKETS OVERVIEW (Rebate Eligible)") + lines.append("-" * 45) + total_fiat_vol = sum(f["total_volume_usdt"] for f in fiat_summary) + lines.append(f"Total FIAT volume: {format_volume(total_fiat_vol)} USDT equiv.") + lines.append("") + for f in fiat_summary: + lines.append( + f" {f['quote']:>4s}: {f['pair_count']:3d} pairs | " + f"Vol: {format_volume(f['total_volume_usdt']):>8s} | " + f"Top: {f['top_pair']}" + ) + + # Focus FIAT detail + if focus_pairs and focus_rate > 0: + filtered = [ + p + for p in focus_pairs + if p["quote_volume_24h"] * focus_rate >= config.min_volume_usd + ] + lines.append("") + lines.append(f"TOP {focus}-QUOTED MARKETS (Rebates)") + lines.append("-" * 45) + for i, p in enumerate(filtered[: config.top_n], 1): + vol_usdt = p["quote_volume_24h"] * focus_rate + chg = p["price_change_pct"] + chg_s = f"+{chg:.1f}%" if chg >= 0 else f"{chg:.1f}%" + lines.append( + f" {i:2d}. {p['trading_pair']:<12s} " + f"Vol: {format_volume(vol_usdt):>8s} " + f"24h: {chg_s:>7s} " + f"Trades: {p['trades_24h']:,}" + ) + + # Top USDT + lines.append("") + lines.append(f"TOP {config.top_n} USDT-QUOTED MARKETS") + lines.append("-" * 45) + for i, p in enumerate(usdt_top, 1): + chg = p["price_change_pct"] + chg_s = f"+{chg:.1f}%" if chg >= 0 else f"{chg:.1f}%" + lines.append( + f" {i:2d}. {p['trading_pair']:<12s} " + f"Vol: {format_volume(p['quote_volume_24h']):>8s} " + f"24h: {chg_s:>7s} " + f"Trades: {p['trades_24h']:,}" + ) + + text = "\n".join(lines) + + # --- Chart: two panels --- + fig = make_subplots( + rows=2, + cols=1, + subplot_titles=[ + f"FIAT Market Volume (USDT equiv.)", + f"Top 30 USDT Markets — Volume vs 24h Change", + ], + row_heights=[0.35, 0.65], + vertical_spacing=0.12, + ) + + # Panel 1: FIAT volume bar chart + if fiat_summary: + quotes = [f["quote"] for f in fiat_summary] + volumes = [f["total_volume_usdt"] for f in fiat_summary] + bar_colors = [QUOTE_COLORS.get(q, "#8b949e") for q in quotes] + fig.add_trace( + go.Bar( + x=quotes, + y=volumes, + marker_color=bar_colors, + text=[format_volume(v) for v in volumes], + textposition="outside", + textfont=dict(size=9, color=COLORS["text"]), + hovertemplate="%{x}: %{y:,.0f} USDT", + showlegend=False, + ), + row=1, + col=1, + ) + + # Panel 2: Scatter — volume vs price change for top USDT + scatter_data = usdt_top[:30] + if scatter_data: + volumes_s = [p["quote_volume_24h"] for p in scatter_data] + changes = [p["price_change_pct"] for p in scatter_data] + labels = [p["base"] for p in scatter_data] + scatter_colors = [COLORS["green"] if c >= 0 else COLORS["red"] for c in changes] + + fig.add_trace( + go.Scatter( + x=changes, + y=volumes_s, + mode="markers+text", + marker=dict( + color=scatter_colors, + size=[max(8, min(25, v / max(volumes_s) * 25)) for v in volumes_s], + line=dict(width=1, color=COLORS["bg_paper"]), + ), + text=labels, + textposition="top center", + textfont=dict(size=8, color=COLORS["text_dim"]), + hovertemplate="%{text}
Vol: %{y:,.0f}
Chg: %{x:.1f}%", + showlegend=False, + ), + row=2, + col=1, + ) + + fig.update_layout( + paper_bgcolor=COLORS["bg_paper"], + plot_bgcolor=COLORS["bg_plot"], + font=dict(color=COLORS["text_dim"], size=10), + margin=dict(l=60, r=30, t=60, b=40), + height=800, + ) + for i in range(1, 3): + fig.update_xaxes(gridcolor=COLORS["grid"], row=i, col=1) + fig.update_yaxes(gridcolor=COLORS["grid"], row=i, col=1) + fig.update_yaxes(type="log", title="Volume (USDT)", row=1, col=1) + fig.update_yaxes(type="log", title="Volume 24h (USDT)", row=2, col=1) + fig.update_xaxes(title="24h Change (%)", row=2, col=1) + + # Annotation styling for subplot titles + for ann in fig.layout.annotations: + ann.font = dict(size=12, color=COLORS["text"]) + + # --- Table data for report --- + table_rows = [] + + # FIAT summary table + for f in fiat_summary: + table_rows.append( + { + "Section": "FIAT Overview", + "Quote": f["quote"], + "Pairs": f["pair_count"], + "Total Volume": format_volume(f["total_volume_usdt"]), + "Top Pair": f["top_pair"], + } + ) + + # Focus pairs table + if focus_pairs and focus_rate > 0: + for p in focus_pairs[: config.top_n]: + vol_usdt = p["quote_volume_24h"] * focus_rate + if vol_usdt < config.min_volume_usd: + continue + table_rows.append( + { + "Section": f"{focus} Markets", + "Pair": p["trading_pair"], + "Volume (USDT)": format_volume(vol_usdt), + "24h Change": f"{p['price_change_pct']:+.1f}%", + "Trades": f"{p['trades_24h']:,}", + } + ) + + # USDT top pairs + for p in usdt_top[: config.top_n]: + table_rows.append( + { + "Section": "USDT Markets", + "Pair": p["trading_pair"], + "Volume (USDT)": format_volume(p["quote_volume_24h"]), + "24h Change": f"{p['price_change_pct']:+.1f}%", + "Trades": f"{p['trades_24h']:,}", + } + ) + + return text, fig, table_rows + + +async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: + """Discover spot markets across quote currencies.""" + # Fetch all tickers + try: + tickers = await fetch_spot_tickers() + except Exception as e: + return RoutineResult(text=f"Failed to fetch Binance spot tickers: {e}") + + if not tickers: + return RoutineResult(text="No ticker data returned from Binance.") + + # Categorize and get rates + by_quote = categorize_tickers(tickers) + usdt_rates = await _fetch_usdt_rates(tickers) + + # Build output + text, fig, table_rows = _build_report(by_quote, usdt_rates, config) + + # Generate report + from condor.reports import ReportBuilder + + # Split tables by section + fiat_overview = [r for r in table_rows if r.get("Section") == "FIAT Overview"] + focus_rows = [ + r + for r in table_rows + if r.get("Section") == f"{config.focus_quote.upper()} Markets" + ] + usdt_rows = [r for r in table_rows if r.get("Section") == "USDT Markets"] + + # Clean section column + for rows in [fiat_overview, focus_rows, usdt_rows]: + for r in rows: + r.pop("Section", None) + + builder = ReportBuilder("Spot Market Discovery") + builder.source("routine", "spot_markets").tags( + ["spot", "markets", "discovery", "fiat", "rebates"] + ) + builder.plotly(fig) + + builder.markdown("### FIAT Markets Overview (Rebate Eligible)") + if fiat_overview: + builder.table(fiat_overview) + + if focus_rows: + builder.markdown(f"### Top {config.focus_quote.upper()}-Quoted Markets") + builder.table(focus_rows) + + builder.markdown("### Top USDT-Quoted Markets") + if usdt_rows: + builder.table(usdt_rows) + + await builder.save() + + return RoutineResult(text=text) diff --git a/agents/brigado/routines/toxic_flow_detector.py b/agents/brigado/routines/toxic_flow_detector.py new file mode 100644 index 000000000..e69de29bb diff --git a/agents/brigado/strategies/brl_mm/strategy.md b/agents/brigado/strategies/brl_mm/strategy.md new file mode 100644 index 000000000..ca1ffb6e1 --- /dev/null +++ b/agents/brigado/strategies/brl_mm/strategy.md @@ -0,0 +1,32 @@ +--- +name: BRL MM +description: BRL market making research agent. Tracks USDT-BRL spreads, BTC/fiat volume + dynamics, and bot performance for the Brazilian market making operation. +agent_key: ollama:qwen3.5:9 +skills: [] +default_config: + connector_name: binance + quote_currency: BRL +default_trading_context: '' +created_by: 481175164 +created_at: '2026-05-04T00:00:00+00:00' +--- + +### Routines + +#### brl_dashboard +Multi-timeframe technical dashboard for USDT-BRL with candlesticks, SMA, RSI, and support/resistance levels across 4 timeframes (1m, 15m, 1h, 1d). + +#### btc_brl_volume +Compares BTC-USDT vs BTC-BRL volume over 7 days and analyzes 1-second price arbitrage spread to detect opportunities. + +#### btc_fiat_markets +Stacked area chart of BTC trading volume across fiat pairs (BRL, ARS, MXN, EUR, JPY, PLN), normalized to USDT. + +#### bot_report +Comprehensive bot performance report with volume, PnL, fees, market share, rebates, and per-controller breakdown. + +### Journal +- Log market observations: BRL premium/discount trends, volume shifts +- Track bot performance metrics over time +- Note arbitrage windows and their duration diff --git a/agents/condor/skills/agent_builder/SKILL.md b/agents/condor/skills/agent_builder/SKILL.md index 7215d5001..418006954 100644 --- a/agents/condor/skills/agent_builder/SKILL.md +++ b/agents/condor/skills/agent_builder/SKILL.md @@ -164,56 +164,23 @@ Only if the user wants the agent to act autonomously. The agent can already loop this step — `start_agent(strategy_id="")` ticks a default playbook driven by its AGENT.md — but that default is deliberately generic. A **strategy** is the specific tick playbook the engine runs in a **session**, and it is what you want for anything that -trades. Make clear the loop does NOT have to trade — define the tick task however the -user wants: - -- read routine X's output and **decide whether to trade** (create/stop executors), -- or just **send a report / notification**, -- or watch a condition and act only when it's met, - -…running at a **frequency the user sets** (`frequency_sec`). - -If the loop creates executors, BEFORE writing the strategy fetch the schema for every -executor type it will use — `manage_executors(executor_type="grid_strike")`, etc. — and -embed the required fields/types directly into the instructions; the tick LLM has no other -way to learn them. Same for any controller config it manages (`manage_controllers`). - -``` -manage_trading_agent( - action="create_strategy", - agent_slug="", # the agent must already exist - name="BRL MM", - description="…", - instructions="", - # agent_key omitted → inherits the owning agent's model; overridable at launch - config={"connector_name": "binance", "frequency_sec": 60, - "total_amount_quote": 100, "execution_mode": "loop"} -) -``` - -Strategy instructions (the tick system prompt) MUST include: **Objective**; **Analysis** -(which routine to call by name and how to read it); **Decision logic** (act / report / -hold); and — only if it trades — an **Executor config** with the FULL schema (every -required field, type, range, ordering rule), **Parameter inference** (how to derive -prices/side/TP from routine output + market data), **Risk rules** (max position, position -limits, stop behaviour), and **Error recovery** (on a failed create, re-fetch the schema, -fix, retry once, journal it). - -**Dry run before live** (if it trades): -``` -manage_trading_agent(action="start_agent", strategy_id="", - config={"execution_mode": "dry_run", "agent_key": "ollama:llama3.1", - "trading_context": "Trade BTC-USDT on binance_perpetual", - "frequency_sec": 60, "total_amount_quote": 100, - "risk_limits": {"max_position_size_quote": 200, "max_open_executors": 3}}) -``` -Review with `trading_agent_journal_read(agent_id=…, section="run:1")`: routines called -right, decision logic sound, conditional language ("would place…"), no real create/stop -calls, risk rules respected. Don't go live until the user is satisfied. - -**Go live:** offer `run_once` (single live tick), `loop` (continuous), or `loop` + -`max_ticks`. Confirm the live model, start, confirm it's running, give monitoring -commands. Always include risk limits when a loop agent can trade. +trades. + +**How a strategy is authored, dry-run and launched lives in the shared `strategy_builder` +playbook — read it (`manage_skill(action="read", name="strategy_builder")`) and follow it, +passing `agent_slug=""`.** It is the single source of truth, and it is shared +precisely so an agent can give *itself* a loop without coming back through you. Don't +restate its mechanics here; your job at this step is only to: + +- decide **with the user** whether a dedicated strategy is warranted at all, +- make clear the loop does NOT have to trade — it can read routine X's output and decide + to trade, send a report, or watch a condition — at a **frequency the user sets** + (`frequency_sec`), +- then run `strategy_builder` for the agent you just created. + +If the agent is capable enough to author its own loop, prefer handing it the job: +`consult(agent="", task="give yourself a loop that …")`. It reads the same +shared playbook and knows its own domain better than you do. ## Monitoring existing agents 1. `manage_trading_agent(action="list_agent_definitions")` — all agents, with their diff --git a/agents/condor/skills/verify_connector_support/SKILL.md b/agents/condor/skills/verify_connector_support/SKILL.md new file mode 100644 index 000000000..db593d367 --- /dev/null +++ b/agents/condor/skills/verify_connector_support/SKILL.md @@ -0,0 +1,22 @@ +--- +name: verify_connector_support +description: Check what a connector actually supports before answering capability + questions +when_to_use: User asks "can I use connector X?" or "does Y support Z?" — any capability + question about a connector or DEX +created: '2026-08-12T11:51:59Z' +source: chat +--- + +## Verify Connector Support + +Never guess connector capabilities from memory. Always pull the authoritative source first. + +### Steps + +1. **LP / CLMM questions** → `manage_executors(executor_type="lp_executor")` — read the "Supported DEXs" section +2. **AMM swap/pool-creation questions** → `manage_amm()` (no action) — read the connector list +3. **Pool discovery questions** → `explore_dex_pools` tool description lists supported connectors +4. **Market data / candle questions** → `get_market_data` guide or the `candles_without_a_candle_feed` skill + +Then answer from what the guide actually says — not from what you remember. diff --git a/agents/delta_neutral_funding_agent/AGENT.md b/agents/delta_neutral_funding_agent/AGENT.md index e1b79387f..07cd0bc07 100644 --- a/agents/delta_neutral_funding_agent/AGENT.md +++ b/agents/delta_neutral_funding_agent/AGENT.md @@ -11,6 +11,8 @@ tools: - manage_bots - manage_routines - search_history +- manage_trading_agent +- trading_agent_journal_read - manage_memory - manage_skill when_to_consult: When the user asks about delta-neutral funding strategies on HIP-3 diff --git a/agents/delta_neutral_funding_agent/routines/hip3_dn_pair_monitor.py b/agents/delta_neutral_funding_agent/routines/hip3_dn_pair_monitor.py index ccdda5457..ed131e1e6 100644 --- a/agents/delta_neutral_funding_agent/routines/hip3_dn_pair_monitor.py +++ b/agents/delta_neutral_funding_agent/routines/hip3_dn_pair_monitor.py @@ -8,6 +8,7 @@ import aiohttp from pydantic import BaseModel, Field from telegram.ext import ContextTypes + from config_manager import get_client logger = logging.getLogger(__name__) diff --git a/agents/delta_neutral_funding_agent/routines/hip3_pairs_backtest.py b/agents/delta_neutral_funding_agent/routines/hip3_pairs_backtest.py index 3e12513b8..0a217535b 100644 --- a/agents/delta_neutral_funding_agent/routines/hip3_pairs_backtest.py +++ b/agents/delta_neutral_funding_agent/routines/hip3_pairs_backtest.py @@ -8,6 +8,7 @@ import aiohttp from pydantic import BaseModel, Field from telegram.ext import ContextTypes + from config_manager import get_client logger = logging.getLogger(__name__) @@ -889,6 +890,7 @@ def _fmt_signal(name: str, r: dict, flags: list) -> str: # ── 7. ReportBuilder ────────────────────────────────────────────────────── import plotly.graph_objects as go + from condor.reports import ReportBuilder builder = ReportBuilder( @@ -1258,6 +1260,7 @@ def _fmt(label, r, beta, r2): # ── 9. ReportBuilder ────────────────────────────────────────────────────── import plotly.graph_objects as go + from condor.reports import ReportBuilder builder = ReportBuilder(f"HIP-3 Pairs Backtest: {issuer_upper}") diff --git a/agents/directional_trader/routines/ema_research_charts.py b/agents/directional_trader/routines/ema_research_charts.py index 45b6a81ef..b6bb01063 100644 --- a/agents/directional_trader/routines/ema_research_charts.py +++ b/agents/directional_trader/routines/ema_research_charts.py @@ -26,27 +26,41 @@ # bars per calendar day for each supported timeframe BARS_PER_DAY = { - "1m": 1440, "3m": 480, "5m": 288, "15m": 96, - "30m": 48, "1h": 24, "4h": 6, "1d": 1, + "1m": 1440, + "3m": 480, + "5m": 288, + "15m": 96, + "30m": 48, + "1h": 24, + "4h": 6, + "1d": 1, } class Config(BaseModel): """EMA trend research — sweep EMA combos, score by Sharpe × persistence, show current signal state.""" - connector: str = Field(default="binance_perpetual", description="Exchange connector") + connector: str = Field( + default="binance_perpetual", description="Exchange connector" + ) pairs: str = Field( default="BTC-USDT,ETH-USDT,SOL-USDT", description="Comma-separated trading pairs", ) - timeframe: str = Field(default="15m", description="Candle interval (1m/3m/15m/1h/4h)") + timeframe: str = Field( + default="15m", description="Candle interval (1m/3m/15m/1h/4h)" + ) days: int = Field(default=30, description="Days of history to fetch") - hold_bars: int = Field(default=5, description="Bars ahead to measure post-signal move") + hold_bars: int = Field( + default=5, description="Bars ahead to measure post-signal move" + ) fast_min: int = Field(default=5, description="Min fast EMA period") fast_max: int = Field(default=21, description="Max fast EMA period") slow_min: int = Field(default=21, description="Min slow EMA period") slow_max: int = Field(default=89, description="Max slow EMA period") - adx_filter: int = Field(default=0, description="Min ADX for a signal to count (0=disabled)") + adx_filter: int = Field( + default=0, description="Min ADX for a signal to count (0=disabled)" + ) top_n: int = Field(default=5, description="Top N combos to show per pair") @@ -308,7 +322,9 @@ def _grid(lo, hi, n=5): # ── Fetch ───────────────────────────────────────────────────────────────── data = {} for pair in pairs: - df = await _fetch_df(client, config.connector, pair, config.timeframe, config.days) + df = await _fetch_df( + client, config.connector, pair, config.timeframe, config.days + ) if df is not None and len(df) >= min_bars: data[pair] = df logger.info(f" {pair}/{config.timeframe}: {len(df)} candles") @@ -344,7 +360,9 @@ def _grid(lo, hi, n=5): "adx_pct_trending": float((adx_s > 25).mean() * 100), "autocorr_lag1": round(autocorr, 4), "regime": ( - "trend" if autocorr > 0.05 else ("mean-rev" if autocorr < -0.05 else "random") + "trend" + if autocorr > 0.05 + else ("mean-rev" if autocorr < -0.05 else "random") ), } @@ -371,9 +389,7 @@ def _grid(lo, hi, n=5): top3 = all_scored[:3] best_per_pair = { - pair: sweep_results[pair][0] - for pair in active_pairs - if sweep_results.get(pair) + pair: sweep_results[pair][0] for pair in active_pairs if sweep_results.get(pair) } current_signals = { pair: { @@ -388,7 +404,9 @@ def _grid(lo, hi, n=5): fig.patch.set_facecolor("#0f172a") fig.suptitle( f"Returns Distribution & Realized Volatility — {config.days}d, {config.timeframe}", - fontsize=13, fontweight="bold", color="white", + fontsize=13, + fontweight="bold", + color="white", ) for ci, pair in enumerate(active_pairs): df = data[pair] @@ -399,21 +417,31 @@ def _grid(lo, hi, n=5): ax_h = axes[0, ci] _style_ax(ax_h) ax_h.hist(ret, bins=100, color=color, alpha=0.75, density=True) - ax_h.axvline(ret.mean(), color="white", linestyle="--", linewidth=1, - label=f"μ={ret.mean():.4f}%") + ax_h.axvline( + ret.mean(), + color="white", + linestyle="--", + linewidth=1, + label=f"μ={ret.mean():.4f}%", + ) ax_h.set_title( f"{pair}\nskew={s['ret_skew']:.2f} kurt={s['ret_kurt']:.2f}", - fontsize=9, color="white", + fontsize=9, + color="white", ) ax_h.set_xlabel("Return %", color="white", fontsize=7) ax_h.legend(fontsize=7, facecolor="#1e293b", labelcolor="white") ax_v = axes[1, ci] _style_ax(ax_v) - rvol = (ret.rolling(bars_per_day * 5).std() * math.sqrt(252 * bars_per_day)).dropna() - dt_slice = df["datetime"].iloc[-len(rvol):] + rvol = ( + ret.rolling(bars_per_day * 5).std() * math.sqrt(252 * bars_per_day) + ).dropna() + dt_slice = df["datetime"].iloc[-len(rvol) :] ax_v.plot(dt_slice.values, rvol.values, color=color, linewidth=0.8) - ax_v.set_title(f"{pair} Ann. RVol\nμ={rvol.mean():.1f}%", fontsize=9, color="white") + ax_v.set_title( + f"{pair} Ann. RVol\nμ={rvol.mean():.1f}%", fontsize=9, color="white" + ) ax_v.set_ylabel("RVol %", color="white", fontsize=7) ax_v.xaxis.set_major_formatter(mdates.DateFormatter("%m/%d")) ax_v.tick_params(axis="x", rotation=45) @@ -431,7 +459,9 @@ def _grid(lo, hi, n=5): fig.patch.set_facecolor("#0f172a") fig.suptitle( f"ADX Distribution — Trend Regime ({config.days}d, {config.timeframe})", - fontsize=13, fontweight="bold", color="white", + fontsize=13, + fontweight="bold", + color="white", ) for ci, pair in enumerate(active_pairs): df = data[pair] @@ -444,12 +474,16 @@ def _grid(lo, hi, n=5): ax.hist(adx_s, bins=60, color=color, alpha=0.75) ax.axvline(25, color="#ef4444", linestyle="--", linewidth=1.2, label="ADX=25") ax.axvline( - float(adx_s.median()), color="white", linestyle="-.", linewidth=1, + float(adx_s.median()), + color="white", + linestyle="-.", + linewidth=1, label=f"median={adx_s.median():.1f}", ) ax.set_title( f"{pair}\n{s['adx_pct_trending']:.0f}% time ADX>25", - fontsize=9, color="white", + fontsize=9, + color="white", ) ax.legend(fontsize=7, facecolor="#1e293b", labelcolor="white") ax.set_xlabel("ADX", color="white", fontsize=7) @@ -467,7 +501,9 @@ def _grid(lo, hi, n=5): fig.patch.set_facecolor("#0f172a") fig.suptitle( f"Best EMA Combo per Pair — Last 500 bars ({config.timeframe})", - fontsize=12, fontweight="bold", color="white", + fontsize=12, + fontweight="bold", + color="white", ) for ri, pair in enumerate(active_pairs): if pair not in best_per_pair: @@ -482,26 +518,59 @@ def _grid(lo, hi, n=5): ax = axes[ri, 0] _style_ax(ax) - ax.plot(df_c["datetime"], df_c["close"], color="white", linewidth=0.6, - label="Close", zorder=2) - ax.plot(df_c["datetime"], df_c["ema_f"], color="#3b82f6", linewidth=1.3, - label=f"EMA{fp}", zorder=3) - ax.plot(df_c["datetime"], df_c["ema_s"], color="#f59e0b", linewidth=1.3, - label=f"EMA{sp}", zorder=3) + ax.plot( + df_c["datetime"], + df_c["close"], + color="white", + linewidth=0.6, + label="Close", + zorder=2, + ) + ax.plot( + df_c["datetime"], + df_c["ema_f"], + color="#3b82f6", + linewidth=1.3, + label=f"EMA{fp}", + zorder=3, + ) + ax.plot( + df_c["datetime"], + df_c["ema_s"], + color="#f59e0b", + linewidth=1.3, + label=f"EMA{sp}", + zorder=3, + ) longs = df_c[df_c["cross"] & (df_c["sig"] == 1)] shorts = df_c[df_c["cross"] & (df_c["sig"] == -1)] - ax.scatter(longs["datetime"], longs["close"], marker="^", color="#22c55e", - s=50, zorder=5, label="Long") - ax.scatter(shorts["datetime"], shorts["close"], marker="v", color="#ef4444", - s=50, zorder=5, label="Short") + ax.scatter( + longs["datetime"], + longs["close"], + marker="^", + color="#22c55e", + s=50, + zorder=5, + label="Long", + ) + ax.scatter( + shorts["datetime"], + shorts["close"], + marker="v", + color="#ef4444", + s=50, + zorder=5, + label="Short", + ) sig_label = "▲ LONG" if best["current_signal"] == 1 else "▼ SHORT" ax.set_title( f"{pair} {config.timeframe} | EMA{fp}/{sp} | score={best['score']:.3f} | " f"win={best['win_rate']:.0f}% | sharpe={best['sharpe']:.2f} | " f"persist={best['avg_persistence']:.0f}b | NOW: {sig_label}", - fontsize=9, color="white", + fontsize=9, + color="white", ) ax.legend(fontsize=8, facecolor="#1e293b", labelcolor="white") ax.xaxis.set_major_formatter(mdates.DateFormatter("%m/%d %H:%M")) @@ -540,7 +609,8 @@ def _grid(lo, hi, n=5): ax.set_ylabel("Fast EMA", color="white", fontsize=9) ax.set_title( f"{pair_hm} — EMA Score Heatmap (Sharpe × Persistence)", - fontsize=11, color="white", + fontsize=11, + color="white", ) cbar = plt.colorbar(im, ax=ax) cbar.ax.tick_params(colors="white") @@ -550,11 +620,18 @@ def _grid(lo, hi, n=5): bfi, bsi = fi.get(best_hm["fast"]), si.get(best_hm["slow"]) if bfi is not None and bsi is not None: ax.add_patch( - plt.Rectangle((bsi - 0.5, bfi - 0.5), 1, 1, fill=False, - edgecolor="white", linewidth=2) + plt.Rectangle( + (bsi - 0.5, bfi - 0.5), + 1, + 1, + fill=False, + edgecolor="white", + linewidth=2, + ) + ) + ax.text( + bsi, bfi, "★", ha="center", va="center", color="white", fontsize=14 ) - ax.text(bsi, bfi, "★", ha="center", va="center", - color="white", fontsize=14) plt.tight_layout() await context.bot.send_photo( @@ -569,7 +646,9 @@ def _grid(lo, hi, n=5): fig.patch.set_facecolor("#0f172a") fig.suptitle( "Normalized Distance: (close − EMA_slow) / ATR(14)", - fontsize=11, fontweight="bold", color="white", + fontsize=11, + fontweight="bold", + color="white", ) for ci, pair in enumerate(active_pairs): if pair not in best_per_pair: @@ -590,7 +669,8 @@ def _grid(lo, hi, n=5): ax.axvline(0, color="white", linestyle="-", linewidth=0.5) ax.set_title( f"{pair} | EMA{sp}\nμ={norm_dist.mean():.2f} σ={norm_dist.std():.2f}", - fontsize=9, color="white", + fontsize=9, + color="white", ) ax.legend(fontsize=7, facecolor="#1e293b", labelcolor="white") ax.set_xlabel("(close − EMA) / ATR", color="white", fontsize=7) @@ -681,12 +761,23 @@ def _grid(lo, hi, n=5): rb.markdown("## Top EMA Combos\n") rb.table( top3, - ["pair", "fast", "slow", "score", "win_rate", "sharpe", - "avg_persistence", "signals_per_day", "current_signal"], + [ + "pair", + "fast", + "slow", + "score", + "win_rate", + "sharpe", + "avg_persistence", + "signals_per_day", + "current_signal", + ], ) if top3: rb.markdown(f"## Recommendation\n\n{lines[-1]}") - rb.tags(["ema", "research", "trend"] + [p.split("-")[0].lower() for p in active_pairs]) + rb.tags( + ["ema", "research", "trend"] + [p.split("-")[0].lower() for p in active_pairs] + ) await rb.save() return summary diff --git a/agents/directional_trader/strategies/ema_trend_loop/strategy.md b/agents/directional_trader/strategies/ema_trend_loop/strategy.md index 02199e638..188ca56bc 100644 --- a/agents/directional_trader/strategies/ema_trend_loop/strategy.md +++ b/agents/directional_trader/strategies/ema_trend_loop/strategy.md @@ -119,6 +119,17 @@ Write a state entry: If you observed anything useful this tick (e.g. a config underperforming vs its backtest, a pair in a choppy regime, a config that caught a clean trend), write a learning entry. These learnings will inform Steps 1–3 on future ticks. +### Step 8 — Notify user +Call `send_notification` with a brief tick summary. Keep it short (4–6 lines): +``` +🤖 EMA Trend Loop — Tick +Bot: +Controllers: | ... +Session PnL: +Action: +``` +Always send this even if nothing changed — the user needs to know the agent is alive. + ## Risk rules - Max global drawdown: 15% of total_amount_quote - Max per-position loss before stopping controller: 10% diff --git a/agents/market_making_expert/AGENT.md b/agents/market_making_expert/AGENT.md index b8efc86b0..0f821f848 100644 --- a/agents/market_making_expert/AGENT.md +++ b/agents/market_making_expert/AGENT.md @@ -10,6 +10,9 @@ tools: - manage_controllers - manage_bots - search_history +- manage_routines +- manage_trading_agent +- trading_agent_journal_read - manage_memory - manage_skill when_to_consult: When the user asks about market regime, whether spreads are appropriate, diff --git a/agents/market_making_expert/routines/hip3_market_scanner.py b/agents/market_making_expert/routines/hip3_market_scanner.py index bb01e2227..a41c85442 100644 --- a/agents/market_making_expert/routines/hip3_market_scanner.py +++ b/agents/market_making_expert/routines/hip3_market_scanner.py @@ -7,6 +7,7 @@ import aiohttp from pydantic import BaseModel, Field from telegram.ext import ContextTypes + from config_manager import get_client logger = logging.getLogger(__name__) diff --git a/agents/market_making_expert/routines/market_analyzer.py b/agents/market_making_expert/routines/market_analyzer.py index 8c0d5baba..f9215531b 100644 --- a/agents/market_making_expert/routines/market_analyzer.py +++ b/agents/market_making_expert/routines/market_analyzer.py @@ -2,10 +2,12 @@ import logging import math + from pydantic import BaseModel, Field from telegram.ext import ContextTypes -from config_manager import get_client + from condor.reports.footprint import build_estimated_footprint_figure, candle_timestamps +from config_manager import get_client logger = logging.getLogger(__name__) @@ -474,6 +476,7 @@ def _norm(c): # Generate report import plotly.graph_objects as go + from condor.reports import ReportBuilder builder = ReportBuilder(f"Market Analysis: {pair}") diff --git a/agents/market_making_expert/routines/mm_dashboard.py b/agents/market_making_expert/routines/mm_dashboard.py index a1c5cd3f2..8f53a950b 100644 --- a/agents/market_making_expert/routines/mm_dashboard.py +++ b/agents/market_making_expert/routines/mm_dashboard.py @@ -17,8 +17,10 @@ import logging from collections import Counter from datetime import datetime, timezone + from pydantic import BaseModel, Field from telegram.ext import ContextTypes + from config_manager import get_client logger = logging.getLogger(__name__) diff --git a/agents/meteora_launch_lp/AGENT.md b/agents/meteora_launch_lp/AGENT.md index 3ddc68d69..6bcfc2c4b 100644 --- a/agents/meteora_launch_lp/AGENT.md +++ b/agents/meteora_launch_lp/AGENT.md @@ -11,6 +11,9 @@ tools: - get_portfolio_overview - get_market_data - send_notification +- manage_routines +- manage_trading_agent +- trading_agent_journal_read - manage_memory - manage_skill when_to_consult: When the user asks whether a freshly-graduated Meteora DAMM v2 pool diff --git a/agents/meteora_launch_lp/routines/damm_v2_scanner.py b/agents/meteora_launch_lp/routines/damm_v2_scanner.py index 52ba7d534..2bf6b3e6a 100644 --- a/agents/meteora_launch_lp/routines/damm_v2_scanner.py +++ b/agents/meteora_launch_lp/routines/damm_v2_scanner.py @@ -14,6 +14,7 @@ The scanner deliberately skips pools with an active fee scheduler (base fee often starts near 99% and decays — a token-launch trap) unless include_launch_pools is set. """ + import logging import aiohttp @@ -39,14 +40,28 @@ class Config(BaseModel): """Rank Meteora DAMM v2 pools by fee yield (fees/TVL) for AMM LP entry.""" - quote_asset: str = Field(default="SOL", description="Quote token to require on one side: SOL, USDC, or USDT") - query: str | None = Field(default=None, description="Optional search (token symbol/name/address), e.g. 'JUP'") - ranking_window: str = Field(default="24h", description="Fee-yield window: 1h, 2h, 4h, 12h, or 24h") + quote_asset: str = Field( + default="SOL", + description="Quote token to require on one side: SOL, USDC, or USDT", + ) + query: str | None = Field( + default=None, + description="Optional search (token symbol/name/address), e.g. 'JUP'", + ) + ranking_window: str = Field( + default="24h", description="Fee-yield window: 1h, 2h, 4h, 12h, or 24h" + ) top_n: int = Field(default=10, description="Number of ranked pools to return") min_tvl_usd: float = Field(default=25000.0, description="Minimum pool TVL in USD") - include_launch_pools: bool = Field(default=False, description="Include active-fee-scheduler (launch) pools") - verified_only: bool = Field(default=True, description="Require both tokens verified") - exclude_pools: list[str] = Field(default=[], description="Pool addresses to exclude (already held)") + include_launch_pools: bool = Field( + default=False, description="Include active-fee-scheduler (launch) pools" + ) + verified_only: bool = Field( + default=True, description="Require both tokens verified" + ) + exclude_pools: list[str] = Field( + default=[], description="Pool addresses to exclude (already held)" + ) async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: @@ -70,7 +85,9 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: try: async with aiohttp.ClientSession() as session: - async with session.get(DAMM_V2_API, params=params, timeout=aiohttp.ClientTimeout(total=20)) as resp: + async with session.get( + DAMM_V2_API, params=params, timeout=aiohttp.ClientTimeout(total=20) + ) as resp: if resp.status != 200: return f"damm_v2_scanner: Meteora DAMM v2 API returned HTTP {resp.status}" payload = await resp.json() @@ -93,50 +110,69 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: cfg = p.get("pool_config", {}) or {} if not config.include_launch_pools and cfg.get("is_fee_scheduler_active"): continue # skip launch pools whose base fee starts ~99% - if config.verified_only and not (tx.get("is_verified") and ty.get("is_verified")): + if config.verified_only and not ( + tx.get("is_verified") and ty.get("is_verified") + ): continue # Base = the side that is NOT the quote asset. base, quote_tok = (ty, tx) if tx.get("address") == quote_mint else (tx, ty) fee_yield = float((p.get("fee_tvl_ratio") or {}).get(window) or 0) - candidates.append({ - "pool": p.get("address"), - "pair": p.get("name") or f"{base.get('symbol','?')}-{quote_tok.get('symbol','?')}", - "base_symbol": base.get("symbol") or base.get("address", "")[:6], - "quote_symbol": quote_tok.get("symbol") or quote, - "base_mint": base.get("address"), - "quote_mint": quote_tok.get("address"), - "base_fee_pct": float(cfg.get("base_fee_pct") or 0), - "tvl": float(p.get("tvl") or 0), - "vol_win": float((p.get("volume") or {}).get(window) or 0), - "fee_yield": fee_yield, - "price": float(p.get("current_price") or 0), - }) + candidates.append( + { + "pool": p.get("address"), + "pair": p.get("name") + or f"{base.get('symbol','?')}-{quote_tok.get('symbol','?')}", + "base_symbol": base.get("symbol") or base.get("address", "")[:6], + "quote_symbol": quote_tok.get("symbol") or quote, + "base_mint": base.get("address"), + "quote_mint": quote_tok.get("address"), + "base_fee_pct": float(cfg.get("base_fee_pct") or 0), + "tvl": float(p.get("tvl") or 0), + "vol_win": float((p.get("volume") or {}).get(window) or 0), + "fee_yield": fee_yield, + "price": float(p.get("current_price") or 0), + } + ) if not candidates: - return (f"damm_v2_scanner: no {quote}-quoted DAMM v2 pools passed the filters " - f"(min TVL ${config.min_tvl_usd:,.0f}, verified_only={config.verified_only}, " - f"launch_pools={config.include_launch_pools}).") + return ( + f"damm_v2_scanner: no {quote}-quoted DAMM v2 pools passed the filters " + f"(min TVL ${config.min_tvl_usd:,.0f}, verified_only={config.verified_only}, " + f"launch_pools={config.include_launch_pools})." + ) candidates.sort(key=lambda c: c["fee_yield"], reverse=True) ranked = candidates[: config.top_n] - columns = ["#", "Pair", "FeeYield", "BaseFee", "TVL", f"Vol{window}", "Price", "Pool", "BaseMint"] + columns = [ + "#", + "Pair", + "FeeYield", + "BaseFee", + "TVL", + f"Vol{window}", + "Price", + "Pool", + "BaseMint", + ] rows = [] for i, c in enumerate(ranked, 1): - rows.append({ - "#": i, - "Pair": c["pair"], - "FeeYield": f"{c['fee_yield'] * 100:.3f}%", - "BaseFee": f"{c['base_fee_pct']:.3f}%", - "TVL": f"${c['tvl']:,.0f}", - f"Vol{window}": f"${c['vol_win']:,.0f}", - "Price": f"{c['price']:.4g}", - "Pool": c["pool"], - "BaseMint": c["base_mint"], - # manage_amm hints (constant across rows): connector=meteora, network=solana-mainnet-beta. - "quote_mint": c["quote_mint"], - }) + rows.append( + { + "#": i, + "Pair": c["pair"], + "FeeYield": f"{c['fee_yield'] * 100:.3f}%", + "BaseFee": f"{c['base_fee_pct']:.3f}%", + "TVL": f"${c['tvl']:,.0f}", + f"Vol{window}": f"${c['vol_win']:,.0f}", + "Price": f"{c['price']:.4g}", + "Pool": c["pool"], + "BaseMint": c["base_mint"], + # manage_amm hints (constant across rows): connector=meteora, network=solana-mainnet-beta. + "quote_mint": c["quote_mint"], + } + ) top = rows[0] summary = ( @@ -150,10 +186,13 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: try: from routines.base import RoutineResult + return RoutineResult(text=summary, table_data=rows, table_columns=columns) except Exception: lines = [summary, ""] for r in rows: - lines.append(f"{r['#']}. {r['Pair']} | yield {r['FeeYield']} | fee {r['BaseFee']} | " - f"TVL {r['TVL']} | pool {r['Pool']} | base {r['BaseMint']}") + lines.append( + f"{r['#']}. {r['Pair']} | yield {r['FeeYield']} | fee {r['BaseFee']} | " + f"TVL {r['TVL']} | pool {r['Pool']} | base {r['BaseMint']}" + ) return "\n".join(lines) diff --git a/agents/meteora_launch_lp/routines/easya_graduation_monitor.py b/agents/meteora_launch_lp/routines/easya_graduation_monitor.py index 1fb196159..602a726e4 100644 --- a/agents/meteora_launch_lp/routines/easya_graduation_monitor.py +++ b/agents/meteora_launch_lp/routines/easya_graduation_monitor.py @@ -14,6 +14,7 @@ Note: early LP here is directional-long the token (you must buy the base token to pair it with SOL), so size small and capped. """ + import logging import time @@ -35,22 +36,40 @@ class Config(BaseModel): """Detect fresh EasyA graduations into Meteora DAMM v2, ranked by fee yield.""" - max_age_hours: float = Field(default=72.0, description="Only pools created within this many hours") - min_tvl_usd: float = Field(default=10000.0, description="Minimum pool TVL in USD (graduation liquidity floor)") - min_vol24h_usd: float = Field(default=3000.0, description="Minimum 24h volume in USD (real post-grad demand)") - verified_only: bool = Field(default=False, description="Require the graduated token to be verified") - require_static_fee: bool = Field(default=True, description="Exclude fee-scheduler pools (EasyA grads are static)") + max_age_hours: float = Field( + default=72.0, description="Only pools created within this many hours" + ) + min_tvl_usd: float = Field( + default=10000.0, + description="Minimum pool TVL in USD (graduation liquidity floor)", + ) + min_vol24h_usd: float = Field( + default=3000.0, description="Minimum 24h volume in USD (real post-grad demand)" + ) + verified_only: bool = Field( + default=False, description="Require the graduated token to be verified" + ) + require_static_fee: bool = Field( + default=True, description="Exclude fee-scheduler pools (EasyA grads are static)" + ) top_n: int = Field(default=15, description="Number of ranked graduations to return") async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: # No created_at sort on the API, so fetch a wide TVL-sorted page and filter locally. # EasyA pools graduate with ~$10k+ TVL, so they are within the top page by TVL. - params = {"page": 1, "page_size": 1000, "sort_by": "tvl:desc", "filter_by": "is_blacklisted=false"} + params = { + "page": 1, + "page_size": 1000, + "sort_by": "tvl:desc", + "filter_by": "is_blacklisted=false", + } try: async with aiohttp.ClientSession() as session: - async with session.get(DAMM_V2_API, params=params, timeout=aiohttp.ClientTimeout(total=25)) as resp: + async with session.get( + DAMM_V2_API, params=params, timeout=aiohttp.ClientTimeout(total=25) + ) as resp: if resp.status != 200: return f"easya_graduation_monitor: Meteora DAMM v2 API returned HTTP {resp.status}" payload = await resp.json() @@ -84,50 +103,70 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: if float((p.get("volume") or {}).get("24h") or 0) < config.min_vol24h_usd: continue cfg = p.get("pool_config", {}) or {} - if config.require_static_fee and (cfg.get("is_fee_scheduler_active") or cfg.get("has_fee_scheduler")): + if config.require_static_fee and ( + cfg.get("is_fee_scheduler_active") or cfg.get("has_fee_scheduler") + ): continue if config.verified_only and not easy_side.get("is_verified"): continue - candidates.append({ - "pool": p.get("address"), - "pair": p.get("name") or f"{easy_side.get('symbol','?')}-SOL", - "token_symbol": easy_side.get("symbol") or easy_side.get("address", "")[:6], - "base_mint": easy_side.get("address"), - "verified": bool(easy_side.get("is_verified")), - "base_fee_pct": float(cfg.get("base_fee_pct") or 0), - "tvl": float(p.get("tvl") or 0), - "vol24h": float((p.get("volume") or {}).get("24h") or 0), - "fee_yield24h": float((p.get("fee_tvl_ratio") or {}).get("24h") or 0), - "age_h": age_h, - "price": float(p.get("current_price") or 0), - }) + candidates.append( + { + "pool": p.get("address"), + "pair": p.get("name") or f"{easy_side.get('symbol','?')}-SOL", + "token_symbol": easy_side.get("symbol") + or easy_side.get("address", "")[:6], + "base_mint": easy_side.get("address"), + "verified": bool(easy_side.get("is_verified")), + "base_fee_pct": float(cfg.get("base_fee_pct") or 0), + "tvl": float(p.get("tvl") or 0), + "vol24h": float((p.get("volume") or {}).get("24h") or 0), + "fee_yield24h": float((p.get("fee_tvl_ratio") or {}).get("24h") or 0), + "age_h": age_h, + "price": float(p.get("current_price") or 0), + } + ) if not candidates: - return (f"easya_graduation_monitor: no EasyA graduations in the last {config.max_age_hours:.0f}h " - f"passed the filters (min TVL ${config.min_tvl_usd:,.0f}, min vol24h ${config.min_vol24h_usd:,.0f}, " - f"verified_only={config.verified_only}).") + return ( + f"easya_graduation_monitor: no EasyA graduations in the last {config.max_age_hours:.0f}h " + f"passed the filters (min TVL ${config.min_tvl_usd:,.0f}, min vol24h ${config.min_vol24h_usd:,.0f}, " + f"verified_only={config.verified_only})." + ) # Rank by traction: 24h fee yield (fees/TVL) — the ones actually earning. candidates.sort(key=lambda c: c["fee_yield24h"], reverse=True) ranked = candidates[: config.top_n] - columns = ["#", "Pair", "Age(h)", "FeeYield", "TVL", "Vol24h", "Verified", "Price", "Pool", "BaseMint"] + columns = [ + "#", + "Pair", + "Age(h)", + "FeeYield", + "TVL", + "Vol24h", + "Verified", + "Price", + "Pool", + "BaseMint", + ] rows = [] for i, c in enumerate(ranked, 1): - rows.append({ - "#": i, - "Pair": c["pair"], - "Age(h)": f"{c['age_h']:.1f}", - "FeeYield": f"{c['fee_yield24h'] * 100:.1f}%", - "TVL": f"${c['tvl']:,.0f}", - "Vol24h": f"${c['vol24h']:,.0f}", - "Verified": "yes" if c["verified"] else "no", - "Price": f"{c['price']:.4g}", - "Pool": c["pool"], - "BaseMint": c["base_mint"], - "quote_mint": SOL_MINT, - }) + rows.append( + { + "#": i, + "Pair": c["pair"], + "Age(h)": f"{c['age_h']:.1f}", + "FeeYield": f"{c['fee_yield24h'] * 100:.1f}%", + "TVL": f"${c['tvl']:,.0f}", + "Vol24h": f"${c['vol24h']:,.0f}", + "Verified": "yes" if c["verified"] else "no", + "Price": f"{c['price']:.4g}", + "Pool": c["pool"], + "BaseMint": c["base_mint"], + "quote_mint": SOL_MINT, + } + ) top = rows[0] summary = ( @@ -141,10 +180,13 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: try: from routines.base import RoutineResult + return RoutineResult(text=summary, table_data=rows, table_columns=columns) except Exception: lines = [summary, ""] for r in rows: - lines.append(f"{r['#']}. {r['Pair']} | age {r['Age(h)']}h | yield {r['FeeYield']} | " - f"TVL {r['TVL']} | vol {r['Vol24h']} | verified {r['Verified']} | pool {r['Pool']}") + lines.append( + f"{r['#']}. {r['Pair']} | age {r['Age(h)']}h | yield {r['FeeYield']} | " + f"TVL {r['TVL']} | vol {r['Vol24h']} | verified {r['Verified']} | pool {r['Pool']}" + ) return "\n".join(lines) diff --git a/agents/meteora_launch_lp/routines/launch_safety_check.py b/agents/meteora_launch_lp/routines/launch_safety_check.py index d7f27d7de..ed228db0e 100644 --- a/agents/meteora_launch_lp/routines/launch_safety_check.py +++ b/agents/meteora_launch_lp/routines/launch_safety_check.py @@ -12,6 +12,7 @@ RPC: defaults to the public mainnet endpoint (rate-limited but fine for occasional checks); override with rpc_url for a private endpoint. No credentials are hardcoded. """ + import logging import aiohttp @@ -39,17 +40,32 @@ def _rpc_url_default(cls, v): # Callers (e.g. strategy configs) may pass an empty string meaning "no private RPC" — # fall back to the public endpoint instead of failing every RPC gate on an invalid URL. return v or DEFAULT_RPC - max_top10_holder_pct: float = Field(default=60.0, description="Fail if top-10 holders exceed this % of supply") - require_mint_renounced: bool = Field(default=True, description="Fail if the token mint authority is not renounced") - require_freeze_disabled: bool = Field(default=True, description="Fail if the freeze authority is not disabled") - require_verified: bool = Field(default=False, description="Fail if the token is not verified") - min_lock_pct: float = Field(default=0.0, description="Require locked+vested liquidity ≥ this % of TVL (0 = off)") + + max_top10_holder_pct: float = Field( + default=60.0, description="Fail if top-10 holders exceed this % of supply" + ) + require_mint_renounced: bool = Field( + default=True, description="Fail if the token mint authority is not renounced" + ) + require_freeze_disabled: bool = Field( + default=True, description="Fail if the freeze authority is not disabled" + ) + require_verified: bool = Field( + default=False, description="Fail if the token is not verified" + ) + min_lock_pct: float = Field( + default=0.0, + description="Require locked+vested liquidity ≥ this % of TVL (0 = off)", + ) min_tvl_usd: float = Field(default=10000.0, description="Fail below this TVL") async def _rpc(session, url, method, params): - async with session.post(url, json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params}, - timeout=aiohttp.ClientTimeout(total=20)) as r: + async with session.post( + url, + json={"jsonrpc": "2.0", "id": 1, "method": method, "params": params}, + timeout=aiohttp.ClientTimeout(total=20), + ) as r: if r.status != 200: raise RuntimeError(f"RPC {method} -> HTTP {r.status}") data = await r.json() @@ -64,8 +80,11 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: # 1. Pool + token metadata from the Meteora DAMM v2 API. try: async with aiohttp.ClientSession() as session: - async with session.get(DAMM_V2_API, params={"query": config.pool_address, "page_size": 1}, - timeout=aiohttp.ClientTimeout(total=20)) as resp: + async with session.get( + DAMM_V2_API, + params={"query": config.pool_address, "page_size": 1}, + timeout=aiohttp.ClientTimeout(total=20), + ) as resp: if resp.status != 200: return f"launch_safety_check: Meteora API HTTP {resp.status}" pool = (await resp.json()).get("data", [{}])[0] @@ -82,45 +101,99 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: cfg = pool.get("pool_config", {}) or {} # Static gates from the API. - gates.append(("tvl", tvl >= config.min_tvl_usd, f"${tvl:,.0f} (min ${config.min_tvl_usd:,.0f})")) + gates.append( + ( + "tvl", + tvl >= config.min_tvl_usd, + f"${tvl:,.0f} (min ${config.min_tvl_usd:,.0f})", + ) + ) if config.require_verified: - gates.append(("verified", bool(base.get("is_verified")), f"{base.get('symbol')} verified={base.get('is_verified')}")) + gates.append( + ( + "verified", + bool(base.get("is_verified")), + f"{base.get('symbol')} verified={base.get('is_verified')}", + ) + ) if config.require_freeze_disabled: - gates.append(("freeze_disabled", bool(base.get("freeze_authority_disabled")), - f"freeze_authority_disabled={base.get('freeze_authority_disabled')}")) + gates.append( + ( + "freeze_disabled", + bool(base.get("freeze_authority_disabled")), + f"freeze_authority_disabled={base.get('freeze_authority_disabled')}", + ) + ) if config.min_lock_pct > 0: locked = float(pool.get("permanent_lock_liquidity") or 0) - vested = sum(float(v or 0) for v in (pool.get("vested_liquidity") or {}).values()) + vested = sum( + float(v or 0) for v in (pool.get("vested_liquidity") or {}).values() + ) lock_pct = (locked + vested) / tvl * 100 if tvl else 0.0 - gates.append(("lp_lock", lock_pct >= config.min_lock_pct, f"{lock_pct:.1f}% locked/vested (min {config.min_lock_pct}%)")) + gates.append( + ( + "lp_lock", + lock_pct >= config.min_lock_pct, + f"{lock_pct:.1f}% locked/vested (min {config.min_lock_pct}%)", + ) + ) # 2. On-chain gates via RPC: mint authority + top-holder concentration. try: async with aiohttp.ClientSession() as session: if config.require_mint_renounced: - info = await _rpc(session, config.rpc_url, "getAccountInfo", - [base_mint, {"encoding": "jsonParsed"}]) - parsed = ((info or {}).get("value") or {}).get("data", {}).get("parsed", {}).get("info", {}) + info = await _rpc( + session, + config.rpc_url, + "getAccountInfo", + [base_mint, {"encoding": "jsonParsed"}], + ) + parsed = ( + ((info or {}).get("value") or {}) + .get("data", {}) + .get("parsed", {}) + .get("info", {}) + ) mint_auth = parsed.get("mintAuthority") - gates.append(("mint_renounced", mint_auth is None, f"mintAuthority={mint_auth or 'null (renounced)'}")) - - supply_res = await _rpc(session, config.rpc_url, "getTokenSupply", [base_mint]) + gates.append( + ( + "mint_renounced", + mint_auth is None, + f"mintAuthority={mint_auth or 'null (renounced)'}", + ) + ) + + supply_res = await _rpc( + session, config.rpc_url, "getTokenSupply", [base_mint] + ) total = float((supply_res or {}).get("value", {}).get("uiAmount") or 0) - largest = await _rpc(session, config.rpc_url, "getTokenLargestAccounts", [base_mint]) + largest = await _rpc( + session, config.rpc_url, "getTokenLargestAccounts", [base_mint] + ) accounts = (largest or {}).get("value", []) top10 = sum(float(a.get("uiAmount") or 0) for a in accounts[:10]) top10_pct = (top10 / total * 100) if total else 100.0 - gates.append(("holder_concentration", top10_pct <= config.max_top10_holder_pct, - f"top-10 hold {top10_pct:.1f}% (max {config.max_top10_holder_pct}%)")) + gates.append( + ( + "holder_concentration", + top10_pct <= config.max_top10_holder_pct, + f"top-10 hold {top10_pct:.1f}% (max {config.max_top10_holder_pct}%)", + ) + ) except Exception as e: gates.append(("rpc_checks", False, f"RPC checks failed: {e}")) passed_all = all(ok for _, ok, _ in gates) - lines = [f"Safety check for {base.get('symbol','?')} ({base_mint}) — pool {config.pool_address}", - f"VERDICT: {'PASS' if passed_all else 'FAIL'}", ""] + lines = [ + f"Safety check for {base.get('symbol','?')} ({base_mint}) — pool {config.pool_address}", + f"VERDICT: {'PASS' if passed_all else 'FAIL'}", + "", + ] for name, ok, detail in gates: lines.append(f" [{'PASS' if ok else 'FAIL'}] {name}: {detail}") lines.append("") - lines.append("NOTE: sellability (honeypot) is NOT checked here — round-trip a SELL and BUY quote via " - "manage_amm(quote_swap) before entering. See the launch_safety_check skill.") + lines.append( + "NOTE: sellability (honeypot) is NOT checked here — round-trip a SELL and BUY quote via " + "manage_amm(quote_swap) before entering. See the launch_safety_check skill." + ) return "\n".join(lines) diff --git a/agents/orca_lp_expert/AGENT.md b/agents/orca_lp_expert/AGENT.md new file mode 100644 index 000000000..f4694fd02 --- /dev/null +++ b/agents/orca_lp_expert/AGENT.md @@ -0,0 +1,66 @@ +--- +name: Orca LP Expert +description: Specialist in concentrated liquidity LP on Orca, focused on tokenized + real-world assets (equities, pre-IPO, commodities) on Solana. +agent_key: claude-code:sonnet +tools: [] +when_to_consult: When the user wants to evaluate, open, manage, or exit an Orca CLMM + LP position — especially on tokenized assets like SPCX, GME, or other RWA tokens. + Also when assessing pool attractiveness, setting price ranges, or deciding whether + an open LP position should hold or exit. +server_required: true +server_name: '' +created_by: 481175164 +created_at: '2026-08-14T16:53:03.026845+00:00' +--- + +# Orca LP Expert + +You are a concentrated liquidity LP specialist for **Orca CLMM pools on Solana**, with a focus on **tokenized real-world assets (RWAs)** — equities, pre-IPO names, commodities, and other off-chain assets brought on-chain (e.g. SPCX, GME, SILV, ZEC bridged assets, etc.). + +## Domain + +You handle: +- Pool discovery and attractiveness assessment (fee tier, bin step, volume, TVL, fee/TVL ratio) +- LP range selection for tokenized assets (wider ranges due to lower liquidity and higher volatility vs crypto-native pairs) +- Impermanent loss analysis and break-even fee estimation +- Position sizing given wallet balances +- Entry / hold / exit decisions for open LP positions +- Comparison of tokenized asset pools vs blue-chip pools (SOL-USDC, cbBTC-USDC) + +You do NOT handle: CEX trading, perpetuals, non-Orca DEXes (refer those to the appropriate agent), or tax advice. + +## Key domain knowledge + +### Orca CLMM mechanics +- Bin step controls tick spacing: lower bin step (1, 4) = tighter range = more fees when in-range but higher IL risk; higher bin step (8, 16, 128) = wider range = less IL risk but fewer fees. +- Fee tiers: 0.01% (stables), 0.04% (blue chips), 0.05% (mid vol), 0.16% (high vol / memecoins / tokenized assets), 1.00% (exotic). +- Active bin = the bin where the current price sits. Only liquidity in the active bin earns fees. +- When the price moves outside your range, you hold 100% of one asset (impermanent loss crystallized) and earn zero fees. + +### Tokenized assets (RWAs) on Solana — LP considerations +- These trade with lower on-chain liquidity than crypto-native pairs → higher slippage and wider true bid/ask. +- Price can gap significantly on market open/close or news events (SPCX, GME track equity sessions). +- Prefer **wider ranges** (bin step 16+) for tokenized equity pairs to avoid being knocked out of range during market hours. +- The 0.16% fee tier is almost always appropriate for these pairs. +- Monitor: if the token is thinly traded, a large swap can move the price through your entire range in one transaction. +- Correlation: some tokenized assets (gold/silver) are correlated; others (individual equities) are idiosyncratic. + +### Position sizing heuristics +- For illiquid / tokenized pairs: keep individual position < 20% of on-chain pool TVL to avoid being your own slippage. +- Pair the tokenized asset against USDC (most common), not SOL — reduces double IL exposure. +- Typical entry split: 50/50 by value at current price, centered on the active bin. + +### Pool quality signals +- fee/TVL ratio (24h) > 0.1% = good yield signal +- Pool volume/TVL > 1x/day = active pool +- Low TVL + low volume = avoid (wide spread, no fees earned) + +## How you answer +- Lead with: **Recommendation** (open / hold / exit / avoid), then rationale. +- Use key: value format for pool metrics, not prose paragraphs. +- Flag IL risk explicitly when opening a position on a volatile or thinly-traded pair. +- When unsure of current pool data, call `explore_dex_pools(connector="orca")` or `explore_geckoterminal` to fetch fresh numbers. +- When checking wallet balances, call `get_portfolio_overview`. +- When opening a position, use `manage_executors(executor_type="lp_executor")` and confirm with the user before executing. + diff --git a/agents/orca_lp_expert/routines/orca_rwa_lp_ranger.py b/agents/orca_lp_expert/routines/orca_rwa_lp_ranger.py new file mode 100644 index 000000000..da10e977f --- /dev/null +++ b/agents/orca_lp_expert/routines/orca_rwa_lp_ranger.py @@ -0,0 +1,468 @@ +"""Orca RWA LP Ranger — picks the top 4 RWA pools and suggests an LP price range from 5m candles. + +Uses https://api.orca.so/v2/solana (Cloudflare-fronted; custom User-Agent required). +GeckoTerminal provides 5-minute OHLCV candles for range computation. +""" + +from __future__ import annotations + +import asyncio +import logging +from datetime import datetime, timezone + +import httpx +import numpy as np +import plotly.graph_objects as go +from plotly.subplots import make_subplots +from pydantic import BaseModel, Field +from telegram.ext import ContextTypes + +from condor.pool_data import fetch_ohlcv +from condor.reports import ReportBuilder +from routines.base import RoutineResult + +logger = logging.getLogger(__name__) + +CATEGORY = "Analysis" + +_ORCA_BASE = "https://api.orca.so/v2/solana" +_USER_AGENT = "condor-dex-browser/1.0 (+https://github.com/hummingbot)" +_ORCA_TIMEOUT = httpx.Timeout(15.0, connect=5.0) + + +class Config(BaseModel): + """Orca RWA LP Ranger — picks the top 4 RWA pools and suggests an LP price range from 5m candles.""" + + min_tvl: float = Field(default=1000.0, description="Minimum pool TVL in USD") + include_symbols: str = Field( + default="SPCX,GME,SILV,TSLA,AAPL,NVDA,COIN,MSFT", + description="Comma-separated RWA token symbols to scan", + ) + candle_limit: int = Field( + default=200, description="Number of 5m candles to fetch per pool (max 500)" + ) + atr_periods: int = Field( + default=14, description="ATR period for LP range calculation" + ) + atr_multiplier: float = Field( + default=3.0, description="ATR multiplier for LP range width" + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _safe_float(val, default: float = 0.0) -> float: + try: + return float(val) if val is not None else default + except (TypeError, ValueError): + return default + + +def _ts_to_dt(ts) -> datetime: + """Convert Orca/GeckoTerminal timestamp (int s or ms) to UTC datetime.""" + ts_f = float(ts) + if ts_f > 1_000_000_000_000: # milliseconds + return datetime.fromtimestamp(ts_f / 1000, tz=timezone.utc) + return datetime.fromtimestamp(ts_f, tz=timezone.utc) # seconds + + +def _compute_atr(highs, lows, closes, period: int) -> float: + h = np.array(highs, dtype=float) + l = np.array(lows, dtype=float) + c = np.array(closes, dtype=float) + prev_c = np.roll(c, 1) + prev_c[0] = c[0] + tr = np.maximum(h - l, np.maximum(np.abs(h - prev_c), np.abs(l - prev_c))) + return float(np.mean(tr[-period:])) + + +def _parse_pool(pool: dict) -> dict | None: + """Extract and normalise metrics from a raw Orca v2 whirlpool row.""" + try: + address = pool.get("address") or "" + if not address: + return None + sym_a = (pool.get("tokenA") or {}).get("symbol") or "?" + sym_b = (pool.get("tokenB") or {}).get("symbol") or "?" + # feeRate is raw basis-points * 100 (e.g. 1600 → 0.16%) + fee_rate_raw = _safe_float(pool.get("feeRate")) + fee_tier = fee_rate_raw / 10_000 + tvl = _safe_float(pool.get("tvlUsdc")) + if tvl <= 0: + return None + stats_24h = (pool.get("stats") or {}).get("24h") or {} + volume_24h = _safe_float(stats_24h.get("volume")) + fees_24h = _safe_float(stats_24h.get("fees")) + if fees_24h == 0.0 and fee_tier > 0.0 and volume_24h > 0.0: + fees_24h = volume_24h * fee_tier + fee_tvl_ratio = fees_24h / tvl * 100 if tvl > 0 else 0.0 + vol_tvl_ratio = volume_24h / tvl if tvl > 0 else 0.0 + score = fee_tvl_ratio * 0.6 + vol_tvl_ratio * 0.4 + addr_short = f"{address[:6]}…{address[-4:]}" if len(address) >= 10 else address + return { + "address": address, + "address_short": addr_short, + "token_a_symbol": sym_a, + "token_b_symbol": sym_b, + "fee_tier": round(fee_tier, 4), + "tvl": round(tvl, 2), + "volume_24h": round(volume_24h, 2), + "fees_24h": round(fees_24h, 4), + "fee_tvl_ratio": round(fee_tvl_ratio, 6), + "vol_tvl_ratio": round(vol_tvl_ratio, 6), + "score": round(score, 6), + } + except Exception as exc: + logger.warning("Pool parse error %s: %s", pool.get("address", "?"), exc) + return None + + +async def _search_symbol( + client: httpx.AsyncClient, sem: asyncio.Semaphore, symbol: str +) -> list[dict]: + async with sem: + try: + r = await client.get("/pools/search", params={"q": symbol, "stats": "24h"}) + r.raise_for_status() + body = r.json() + rows = ( + body.get("data", []) + if isinstance(body, dict) + else (body if isinstance(body, list) else []) + ) + sym_upper = symbol.upper() + return [ + p + for p in rows + if isinstance(p, dict) + and ( + ((p.get("tokenA") or {}).get("symbol") or "") + .upper() + .startswith(sym_upper) + or ((p.get("tokenB") or {}).get("symbol") or "") + .upper() + .startswith(sym_upper) + ) + ] + except Exception as exc: + logger.warning("Orca search '%s' failed: %s", symbol, exc) + return [] + + +# --------------------------------------------------------------------------- +# Main +# --------------------------------------------------------------------------- + + +async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: + symbols = [ + s.strip().upper() for s in config.include_symbols.split(",") if s.strip() + ] + + # ── Step 1: Fetch & rank RWA pools ──────────────────────────────────────── + sem = asyncio.Semaphore(5) + async with httpx.AsyncClient( + base_url=_ORCA_BASE, + timeout=_ORCA_TIMEOUT, + headers={"User-Agent": _USER_AGENT, "Accept": "application/json"}, + ) as orca: + batches = await asyncio.gather( + *[_search_symbol(orca, sem, sym) for sym in symbols] + ) + + seen: set[str] = set() + all_pools: list[dict] = [] + for batch in batches: + for raw in batch: + parsed = _parse_pool(raw) + if parsed and parsed["address"] not in seen: + seen.add(parsed["address"]) + all_pools.append(parsed) + + filtered = [p for p in all_pools if p["tvl"] >= config.min_tvl] + filtered.sort(key=lambda x: x["score"], reverse=True) + top4 = filtered[:4] + + if not top4: + builder = ReportBuilder("Orca RWA LP Ranger") + builder.source("routine", "orca_rwa_lp_ranger") + builder.tags(["orca", "rwa", "lp"]) + builder.section("RESULT", None) + builder.markdown( + f"No RWA pools found above min_tvl=${config.min_tvl:,.0f}. " + f"Scanned {len(all_pools)} pools across {len(symbols)} symbols." + ) + builder.manual_order() + await builder.save() + return RoutineResult( + text=( + f"No RWA pools found above min_tvl=${config.min_tvl:,.0f}. " + f"Scanned {len(all_pools)} pools." + ) + ) + + # ── Step 2: Fetch 5m candles ────────────────────────────────────────────── + # Through pool_data, not a raw URL: GeckoTerminal's ~30 req/min is per *IP*, + # shared with the dashboard and the candle poll loop. fetch_ohlcv carries the + # rate gate, the circuit breaker and the single-flight, so no manual pacing. + candle_data: dict[str, list] = {} + for pool in top4: + addr = pool["address"] + ohlcv, err = await fetch_ohlcv( + addr, + "solana", + timeframe="5m", + currency="usd", + limit=config.candle_limit, + ) + if err or not ohlcv: + logger.warning("GeckoTerminal candles for %s failed: %s", addr, err) + candle_data[addr] = [] + continue + candle_data[addr] = sorted(ohlcv, key=lambda x: x[0]) # ascending by timestamp + + # ── Steps 3 & 4: ATR ranges + 2×2 chart ────────────────────────────────── + subplot_titles = [f"{p['token_a_symbol']}-{p['token_b_symbol']}" for p in top4] + while len(subplot_titles) < 4: + subplot_titles.append("") + + fig = make_subplots( + rows=2, + cols=2, + shared_xaxes=False, + subplot_titles=subplot_titles, + ) + + range_table_data: list[dict] = [] + + for i, pool in enumerate(top4): + row = i // 2 + 1 + col = i % 2 + 1 + addr = pool["address"] + ohlcv = candle_data.get(addr, []) + ta = pool["token_a_symbol"] + tb = pool["token_b_symbol"] + + if len(ohlcv) < config.atr_periods + 1: + logger.warning("Insufficient candles for %s-%s: %d", ta, tb, len(ohlcv)) + range_table_data.append( + { + "Pool": f"{ta}-{tb}", + "Current Price": "N/A", + "Lower": "—", + "Upper": "—", + "Range %": "—", + "ATR": "—", + "Fee Tier": f"{pool['fee_tier']:.4f}%", + "TVL": f"${pool['tvl']:,.0f}", + "24h Fees": f"${pool['fees_24h']:,.2f}", + } + ) + continue + + timestamps = [_ts_to_dt(c[0]) for c in ohlcv] + opens = [c[1] for c in ohlcv] + highs = [c[2] for c in ohlcv] + lows = [c[3] for c in ohlcv] + closes = [c[4] for c in ohlcv] + + atr = _compute_atr(highs, lows, closes, config.atr_periods) + current_price = closes[-1] + lower_price = max( + current_price - config.atr_multiplier * atr, current_price * 0.01 + ) + upper_price = current_price + config.atr_multiplier * atr + range_pct = (upper_price - lower_price) / current_price * 100 + + pool["current_price"] = current_price + pool["lower_price"] = lower_price + pool["upper_price"] = upper_price + pool["range_pct"] = range_pct + pool["atr"] = atr + + # Candlestick + fig.add_trace( + go.Candlestick( + x=timestamps, + open=opens, + high=highs, + low=lows, + close=closes, + name=f"{ta}-{tb}", + increasing_line_color="#22c55e", + decreasing_line_color="#ef4444", + showlegend=False, + ), + row=row, + col=col, + ) + + # LP range shaded band + x_band = timestamps + timestamps[::-1] + y_band = [upper_price] * len(timestamps) + [lower_price] * len(timestamps) + fig.add_trace( + go.Scatter( + x=x_band, + y=y_band, + fill="toself", + fillcolor="rgba(99, 102, 241, 0.15)", + line=dict(width=0), + mode="lines", + name="LP Range", + showlegend=(i == 0), + hoverinfo="skip", + ), + row=row, + col=col, + ) + + # Upper bound dashed line + fig.add_shape( + type="line", + x0=timestamps[0], + x1=timestamps[-1], + y0=upper_price, + y1=upper_price, + line=dict(color="#6366f1", width=1.5, dash="dash"), + row=row, + col=col, + ) + # Lower bound dashed line + fig.add_shape( + type="line", + x0=timestamps[0], + x1=timestamps[-1], + y0=lower_price, + y1=lower_price, + line=dict(color="#6366f1", width=1.5, dash="dash"), + row=row, + col=col, + ) + # Current price dotted line + fig.add_shape( + type="line", + x0=timestamps[0], + x1=timestamps[-1], + y0=current_price, + y1=current_price, + line=dict(color="#f59e0b", width=1, dash="dot"), + row=row, + col=col, + ) + + # Price labels at right edge + fig.add_annotation( + x=timestamps[-1], + y=upper_price, + text=f" ${upper_price:.4f}", + showarrow=False, + font=dict(color="#6366f1", size=10), + xanchor="left", + row=row, + col=col, + ) + fig.add_annotation( + x=timestamps[-1], + y=lower_price, + text=f" ${lower_price:.4f}", + showarrow=False, + font=dict(color="#6366f1", size=10), + xanchor="left", + row=row, + col=col, + ) + + fig.update_xaxes(type="date", rangeslider_visible=False, row=row, col=col) + + range_table_data.append( + { + "Pool": f"{ta}-{tb}", + "Current Price": f"${current_price:.4f}", + "Lower": f"${lower_price:.4f}", + "Upper": f"${upper_price:.4f}", + "Range %": f"{range_pct:.1f}%", + "ATR": f"{atr:.6f}", + "Fee Tier": f"{pool['fee_tier']:.4f}%", + "TVL": f"${pool['tvl']:,.0f}", + "24h Fees": f"${pool['fees_24h']:,.2f}", + } + ) + + fig.update_layout( + title="Orca RWA — Top 4 Pools LP Range Suggestion (5m candles)", + template="plotly_dark", + height=800, + legend=dict(orientation="h", yanchor="top", y=-0.05, xanchor="center", x=0.5), + ) + + # ── Step 5: ReportBuilder ───────────────────────────────────────────────── + builder = ReportBuilder("Orca RWA LP Ranger") + builder.source("routine", "orca_rwa_lp_ranger") + builder.tags(["orca", "rwa", "lp", "range", "candles"]) + builder.manual_order() + + builder.section( + "TOP 4 RWA POOLS", "Ranked by composite score (fee/TVL × 0.6 + vol/TVL × 0.4)" + ) + builder.kpi("Pools Analyzed", str(len(all_pools))) + builder.kpi("Pools Selected", str(len(top4))) + for p in top4: + builder.kpi( + f"{p['token_a_symbol']}-{p['token_b_symbol']}", f"Score: {p['score']:.3f}" + ) + + builder.section( + "LP RANGE CHART", "5m candles with ATR-based LP range overlay (purple band)" + ) + builder.plotly(fig) + + builder.section("RANGE DETAILS", "Suggested ranges per pool") + builder.table( + range_table_data, + [ + "Pool", + "Current Price", + "Lower", + "Upper", + "Range %", + "ATR", + "Fee Tier", + "TVL", + "24h Fees", + ], + ) + + builder.section("NOTES", None) + builder.markdown( + f"**Range logic:** ATR({config.atr_periods}) × {config.atr_multiplier} centered on current price\n\n" + "**Candles:** 5m from GeckoTerminal\n\n" + "**Warning:** Tokenized RWAs track equity market hours — range may widen significantly at " + "open/close. Consider widening range further for overnight positions." + ) + + report_id = await builder.save() + + # ── Step 6: RoutineResult ───────────────────────────────────────────────── + pools_with_ranges = [p for p in top4 if "current_price" in p] + return RoutineResult( + text=( + f"Top {len(top4)} RWA pools ranked. " + f"Ranges computed from ATR({config.atr_periods})×{config.atr_multiplier}. " + f"Report: {report_id}" + ), + table_data=[ + { + "Pool": f"{p['token_a_symbol']}-{p['token_b_symbol']}", + "Current": f"${p['current_price']:.4f}", + "Lower": f"${p['lower_price']:.4f}", + "Upper": f"${p['upper_price']:.4f}", + "Range %": f"{p['range_pct']:.1f}%", + "Score": f"{p['score']:.3f}", + } + for p in pools_with_ranges + ], + table_columns=["Pool", "Current", "Lower", "Upper", "Range %", "Score"], + ) diff --git a/agents/orca_lp_expert/routines/orca_rwa_pool_analysis.py b/agents/orca_lp_expert/routines/orca_rwa_pool_analysis.py new file mode 100644 index 000000000..02e969999 --- /dev/null +++ b/agents/orca_lp_expert/routines/orca_rwa_pool_analysis.py @@ -0,0 +1,274 @@ +"""Orca RWA Pool Analysis — scans Orca CLMM for tokenized real-world asset pools +and ranks them by fee yield. + +Uses https://api.orca.so/v2/solana directly (same base as condor/orca_api.py). +Cloudflare-fronted: a custom User-Agent header is required or the request is 403'd. +""" + +from __future__ import annotations + +import asyncio +import logging + +import httpx +from pydantic import BaseModel, Field +from telegram.ext import ContextTypes + +from condor.reports import ReportBuilder +from routines.base import RoutineResult + +logger = logging.getLogger(__name__) + +CATEGORY = "Analysis" + +_BASE_URL = "https://api.orca.so/v2/solana" +_USER_AGENT = "condor-dex-browser/1.0 (+https://github.com/hummingbot)" +_TIMEOUT = httpx.Timeout(15.0, connect=5.0) + + +class Config(BaseModel): + """Orca RWA Pool Analysis — scans Orca CLMM for tokenized real-world asset pools and ranks them by fee yield.""" + + min_tvl: float = Field( + default=1000.0, description="Minimum TVL in USD to include a pool" + ) + include_symbols: str = Field( + default="SPCX,GME,SILV,TSLA,AAPL,NVDA,COIN,MSFT", + description="Comma-separated RWA token symbols to search", + ) + top_n: int = Field(default=10, description="Number of top pools to show in summary") + + +# --------------------------------------------------------------------------- +# Fetch helpers +# --------------------------------------------------------------------------- + + +async def _search_symbol( + client: httpx.AsyncClient, sem: asyncio.Semaphore, symbol: str +) -> list[dict]: + """Search Orca for pools containing a given RWA symbol and filter to close matches.""" + async with sem: + try: + r = await client.get("/pools/search", params={"q": symbol, "stats": "24h"}) + r.raise_for_status() + body = r.json() + rows = ( + body.get("data", []) + if isinstance(body, dict) + else (body if isinstance(body, list) else []) + ) + sym_upper = symbol.upper() + filtered = [] + for p in rows: + if not isinstance(p, dict): + continue + sym_a = ((p.get("tokenA") or {}).get("symbol") or "").upper() + sym_b = ((p.get("tokenB") or {}).get("symbol") or "").upper() + # Accept only when a token symbol starts with the searched RWA symbol. + # startswith avoids false positives like "COIN" matching "BITCOIN". + if sym_a.startswith(sym_upper) or sym_b.startswith(sym_upper): + filtered.append(p) + return filtered + except Exception as exc: + logger.warning("Orca search for %s failed: %s", symbol, exc) + return [] + + +def _parse_pool(pool: dict) -> dict | None: + """Extract and compute metrics from a raw Orca whirlpool row. + + Returns None if the pool has no TVL or fails to parse. + """ + try: + address = pool.get("address") or "" + sym_a = (pool.get("tokenA") or {}).get("symbol") or "?" + sym_b = (pool.get("tokenB") or {}).get("symbol") or "?" + fee_rate_raw = float(pool.get("feeRate") or 0) + fee_tier = fee_rate_raw / 10000 # e.g. 1600 → 0.16 (percent) + tick_spacing = int(pool.get("tickSpacing") or 0) + tvl = float(pool.get("tvlUsdc") or 0) + if tvl <= 0: + return None + stats_24h = (pool.get("stats") or {}).get("24h") or {} + volume_24h = float(stats_24h.get("volume") or 0) + fees_24h = float(stats_24h.get("fees") or 0) + + fee_tvl_ratio = fees_24h / tvl * 100 if tvl > 0 else 0.0 + vol_tvl_ratio = volume_24h / tvl if tvl > 0 else 0.0 + score = fee_tvl_ratio * 0.6 + vol_tvl_ratio * 0.4 + + addr_short = ( + f"{address[:6]}...{address[-4:]}" if len(address) >= 10 else address + ) + + return { + "address": address, + "address_short": addr_short, + "token_a": sym_a, + "token_b": sym_b, + "pool_label": f"{sym_a}-{sym_b} ({addr_short})", + "fee_tier": round(fee_tier, 4), + "tick_spacing": tick_spacing, + "tvl": round(tvl, 2), + "volume_24h": round(volume_24h, 2), + "fees_24h": round(fees_24h, 4), + "fee_tvl_ratio": round(fee_tvl_ratio, 6), + "vol_tvl_ratio": round(vol_tvl_ratio, 6), + "score": round(score, 6), + } + except Exception as exc: + logger.warning("Failed to parse pool %s: %s", pool.get("address", "?"), exc) + return None + + +# --------------------------------------------------------------------------- +# Main entry point +# --------------------------------------------------------------------------- + + +async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: + symbols = [ + s.strip().upper() for s in config.include_symbols.split(",") if s.strip() + ] + + # --- Fetch: parallel per-symbol searches with rate-limiting --- + async with httpx.AsyncClient( + base_url=_BASE_URL, + timeout=_TIMEOUT, + headers={"User-Agent": _USER_AGENT, "Accept": "application/json"}, + ) as http: + sem = asyncio.Semaphore(5) + batches = await asyncio.gather( + *[_search_symbol(http, sem, sym) for sym in symbols] + ) + + # --- Deduplicate by pool address --- + seen: set[str] = set() + raw_pools: list[dict] = [] + for batch in batches: + for p in batch: + addr = p.get("address") or "" + if addr and addr not in seen: + seen.add(addr) + raw_pools.append(p) + + # --- Parse + filter by min_tvl --- + pools: list[dict] = [] + for p in raw_pools: + m = _parse_pool(p) + if m and m["tvl"] >= config.min_tvl: + pools.append(m) + + if not pools: + return RoutineResult( + text=f"No Orca RWA pools found above ${config.min_tvl:,.0f} TVL for symbols: {', '.join(symbols)}" + ) + + # Sort descending by score + pools.sort(key=lambda x: x["score"], reverse=True) + + # --- Aggregate KPIs --- + total_tvl = sum(p["tvl"] for p in pools) + total_vol = sum(p["volume_24h"] for p in pools) + best_fee_tvl = max(p["fee_tvl_ratio"] for p in pools) + + # --- Risk flags (computed before report) --- + thin = [p for p in pools if p["tvl"] < 10_000] + wrong_fee = [p for p in pools if p["fee_tier"] < 0.1] + stagnant = [p for p in pools if p["vol_tvl_ratio"] < 0.05] + + risk_lines = ["## Risk Flags\n"] + if thin: + risk_lines.append("### Thin Liquidity (TVL < $10,000)") + for p in thin: + risk_lines.append( + f"- **{p['token_a']}/{p['token_b']}** (`{p['address_short']}`): TVL = ${p['tvl']:,.2f}" + ) + if wrong_fee: + risk_lines.append("\n### Low Fee Tier for RWA (fee < 0.1%)") + for p in wrong_fee: + risk_lines.append( + f"- **{p['token_a']}/{p['token_b']}** (`{p['address_short']}`): fee = {p['fee_tier']:.4f}%" + ) + if stagnant: + risk_lines.append("\n### Stagnant Pools (vol/TVL < 0.05)") + for p in stagnant: + risk_lines.append( + f"- **{p['token_a']}/{p['token_b']}** (`{p['address_short']}`): vol/TVL = {p['vol_tvl_ratio']:.4f}" + ) + if not thin and not wrong_fee and not stagnant: + risk_lines.append("_No risk flags._") + + # --- Build report --- + builder = ReportBuilder("Orca RWA Pool Analysis") + builder.source("routine", "orca_rwa_pool_analysis") + builder.tags(["orca", "rwa", "lp", "tokenized-assets"]) + + builder.section("PROTOCOL OVERVIEW") + builder.kpi("Total RWA Pools Found", str(len(pools))) + builder.kpi("Total TVL", f"${total_tvl:,.2f}") + builder.kpi("Total 24h Volume", f"${total_vol:,.2f}") + builder.kpi("Best Fee/TVL Ratio", f"{best_fee_tvl:.4f}%") + + builder.section("RWA POOL RANKINGS") + builder.dataset("pools", pools) + builder.data_table( + "pools", + title="All RWA Pools", + columns=[ + "token_a", + "token_b", + "fee_tier", + "tick_spacing", + "tvl", + "volume_24h", + "fees_24h", + "fee_tvl_ratio", + "vol_tvl_ratio", + "score", + "address_short", + ], + ) + + builder.section("POOL YIELD CHART") + builder.chart( + "bar", + "Fee Yield by Pool (fee/TVL %)", + "pools", + "pool_label", + "fee_tvl_ratio", + x_label="Pool", + y_label="Fee/TVL Ratio (%)", + ) + + builder.section("RISK FLAGS") + builder.markdown("\n".join(risk_lines)) + + builder.manual_order() + await builder.save() + + # --- Inline summary: top N --- + top = pools[: config.top_n] + table_data = [ + { + "Pool": f"{p['token_a']}/{p['token_b']}", + "Fee%": f"{p['fee_tier']:.4f}%", + "TVL": f"${p['tvl']:,.2f}", + "Vol 24h": f"${p['volume_24h']:,.2f}", + "Fee/TVL%": f"{p['fee_tvl_ratio']:.4f}%", + "Score": f"{p['score']:.4f}", + } + for p in top + ] + + summary = ( + f"Found {len(pools)} Orca RWA pools across {len(symbols)} symbols — " + f"best Fee/TVL: {best_fee_tvl:.4f}% | Total TVL: ${total_tvl:,.2f}" + ) + + return RoutineResult( + text=summary, + table_data=table_data, + table_columns=["Pool", "Fee%", "TVL", "Vol 24h", "Fee/TVL%", "Score"], + ) diff --git a/agents/orca_lp_expert/skills/add_token_to_gateway/SKILL.md b/agents/orca_lp_expert/skills/add_token_to_gateway/SKILL.md new file mode 100644 index 000000000..f9d3336ff --- /dev/null +++ b/agents/orca_lp_expert/skills/add_token_to_gateway/SKILL.md @@ -0,0 +1,117 @@ +--- +name: add_token_to_gateway +description: Playbook for adding a missing token to the Hummingbot Gateway so it can + be used in Orca LP positions. Covers both the UI walkthrough and the two input formats. +when_to_use: When the user wants to LP or trade a token on Orca but the token is not + in the Gateway token list — Gateway refuses the pair, the executor fails with "token + not found", or the pool browser cannot resolve the token symbol. Common for newly-launched + RWA tokens (SPCX, SILV, GME, etc.) that are not pre-loaded. +created: '2026-08-14T17:00:52Z' +source: agent:orca_lp_expert +--- + +# Add a Token to the Gateway + +Use this playbook whenever a token is not recognized by the Gateway — the pool exists on Orca, but the executor or swap UI cannot resolve the token symbol or mint address. + +--- + +## 1. Confirm the token is really missing + +Before adding, verify the pool exists and find the exact mint address: + +- Run the pool scanner (`orca_rwa_pool_analysis`) or search on Orca UI. +- Note the **mint address** of the missing token (base58, ~44 characters on Solana). +- Note the **network** — for Orca it is always `solana-mainnet-beta` (the Gateway network ID for Solana mainnet). + +--- + +## 2. Navigate to Gateway → Tokens + +In Telegram (or the Condor web dashboard): + +``` +/gateway → 🪙 Tokens → Select network → ➕ Add Token +``` + +> If the network list shows only "default" networks and Solana is not there, tap **🌐 All Networks** to expand it. + +--- + +## 3. Add the token — two input formats + +The bot prompts for input after tapping ➕ Add Token. You have two options: + +### Option A — Address only (recommended for RWAs) +Paste **only the mint address**. The bot fetches symbol, decimals, and name automatically from GeckoTerminal. + +``` +9QFfgxdSqH5zT7j6rZb1y6SZhw2aFtcQu2r6BuYpump +``` + +This works as long as the token is indexed on GeckoTerminal for the Solana network. Most traded RWA tokens (SPCX, SILV, GME, etc.) are indexed. If GeckoTerminal lookup fails, fall back to Option B. + +### Option B — Full format +Send a comma-separated string when auto-lookup fails or you want to override the metadata: + +``` +address,symbol,decimals,name +``` + +Example: +``` +9QFfgxdSqH5zT7j6rZb1y6SZhw2aFtcQu2r6BuYpump,SPCX,6,SpaceX Tokenized Equity +``` + +Field notes: +- `address`: Solana mint address (base58) +- `symbol`: Short ticker as shown on Orca (must match the pool's token symbol) +- `decimals`: Token precision — **6** for most SPL stablecoins and RWA tokens, **9** for native SOL-like tokens. Check on-chain or GeckoTerminal if unsure. +- `name`: Optional — can be omitted (`address,symbol,decimals`) + +--- + +## 4. Verify the token was added + +After success, the bot shows `✅ Token Added Successfully` and returns to the token list for that network. The new token should appear in the grid. + +To confirm it is now usable: +- Go to the pool browser (`/lp` or Dex page) and search for the pair (e.g., `SPCX-USDC`). +- If the pair resolves and shows a pool address, the token is live in Gateway. + +--- + +## 5. Edit or remove a token + +If the metadata was wrong (wrong symbol, wrong decimals): + +``` +/gateway → 🪙 Tokens → Select network → [tap the token symbol] → ✏️ Edit +``` + +Send the corrected values in `symbol,decimals,name` format (address stays the same — Gateway does a delete + re-add internally). + +To remove: tap **🗑 Remove** and confirm. + +--- + +## 6. Network ID reference (Orca = Solana) + +| Network | Gateway ID | +|---------|-----------| +| Solana Mainnet | `solana-mainnet-beta` | +| Ethereum | `ethereum-mainnet` | +| Arbitrum | `arbitrum-mainnet` | +| Base | `base-mainnet` | +| Polygon | `polygon-mainnet` | + +For Orca CLMM LP, always use `solana-mainnet-beta`. + +--- + +## 7. RWA-specific notes + +- Most tokenized equity / RWA tokens on Solana use **6 decimals** — verify before adding. +- If a token is freshly launched and not yet on GeckoTerminal, it may not have a pool on Orca either — check that a pool exists first. +- The token symbol in Gateway must match the symbol the Orca connector uses in the trading pair (e.g., `SPCX-USDC` → symbol must be `SPCX`). A mismatch will cause the executor to fail even after adding. +- After adding a new token, **restart the bot session** if the connector was already running — it caches the token list on startup. diff --git a/agents/smart_money_flow/routines/onchain_flow.py b/agents/smart_money_flow/routines/onchain_flow.py index dd8b152bd..d18f2aacc 100644 --- a/agents/smart_money_flow/routines/onchain_flow.py +++ b/agents/smart_money_flow/routines/onchain_flow.py @@ -17,10 +17,13 @@ Execution venue is independent of this signal: pair the read with Derive perps (`derive_perpetual`) — or any perp connector — via the agent's trading context. """ + import asyncio import logging + from pydantic import BaseModel, Field from telegram.ext import ContextTypes + from config_manager import get_client logger = logging.getLogger(__name__) @@ -29,7 +32,6 @@ # Keyless endpoints (CoinGecko + GeckoTerminal free tier, public Solana RPC). CG = "https://api.coingecko.com/api/v3" -GECKO = "https://api.geckoterminal.com/api/v2" XRPL_NODES = [ "https://xrplcluster.com/", "https://s1.ripple.com:51234", @@ -47,6 +49,7 @@ class Config(BaseModel): """Compute the Smart-Money Flow composite: cross-market + Solana on-chain pulse.""" + context_assets: str = Field( default="bitcoin,ethereum,solana", description="Comma-separated CoinGecko ids for regime + per-asset flow read", @@ -63,11 +66,14 @@ class Config(BaseModel): default="", description="OPTIONAL comma-separated XRPL addresses to watch (empty = skip)", ) - top_n_trending: int = Field(default=7, description="Trending coins to factor into momentum") + top_n_trending: int = Field( + default=7, description="Trending coins to factor into momentum" + ) # ── helpers ──────────────────────────────────────────────────────────────── + async def _get_json(url: str, timeout: float = 12) -> "dict | list | None": import httpx @@ -83,6 +89,22 @@ async def _get_json(url: str, timeout: float = 12) -> "dict | list | None": return None +async def _gecko_json(path: str, params: dict | None = None) -> "dict | None": + """A GeckoTerminal call on the process-wide budget. + + Separate from ``_get_json`` because only this host is rate-limited per IP and + shared with the dashboard; CoinGecko keeps the plain fetcher. Returns None on + any failure, like its sibling — a tick must never raise. + """ + from condor.pool_data import gecko_request + + try: + return await gecko_request("GET", path, params=params) + except Exception as exc: # never raise into a tick + logger.warning("flow: gecko %s failed: %s", path, type(exc).__name__) + return None + + def _clamp(x: float, lo: float = -1.0, hi: float = 1.0) -> float: return max(lo, min(hi, x)) @@ -96,6 +118,7 @@ def _num(v) -> float: # ── raw fetchers (pure async, importable) ────────────────────────────────── + async def fetch_global() -> "dict | list | None": return await _get_json(f"{CG}/global") @@ -125,7 +148,10 @@ async def fetch_solana_pulse(top_n: int = 10) -> dict | None: read. Solana carries materially deeper on-chain liquidity than XRPL, so it is the DEFAULT on-chain signal source. """ - data = await _get_json(f"{GECKO}/networks/solana/tokens/{SOL_MINT}/pools?page=1&include=base_token,dex") + data = await _gecko_json( + f"networks/solana/tokens/{SOL_MINT}/pools", + {"page": 1, "include": "base_token,dex"}, + ) if not data: return None pools = [] @@ -135,13 +161,15 @@ async def fetch_solana_pulse(top_n: int = 10) -> dict | None: tvl = _num(a.get("reserve_in_usd")) if tvl < 50_000: # skip dust/illiquid pools (meaningless momentum) continue - pools.append({ - "name": a.get("name", "?"), - "dex": a.get("dex_id", ""), - "vol24h": vol, - "chg24h": _num(a.get("price_change_percentage", {}).get("h24")), - "tvl": tvl, - }) + pools.append( + { + "name": a.get("name", "?"), + "dex": a.get("dex_id", ""), + "vol24h": vol, + "chg24h": _num(a.get("price_change_percentage", {}).get("h24")), + "tvl": tvl, + } + ) if not pools: return None total_vol = sum(p["vol24h"] for p in pools) @@ -165,8 +193,12 @@ async def fetch_xrpl_amm_pulse(issuer: str) -> dict | None: return None payload = { "method": "amm_info", - "params": [{"asset": {"currency": "XRP"}, - "asset2": {"currency": "RLUSD", "issuer": issuer}}], + "params": [ + { + "asset": {"currency": "XRP"}, + "asset2": {"currency": "RLUSD", "issuer": issuer}, + } + ], } import httpx @@ -185,18 +217,26 @@ async def fetch_xrpl_amm_pulse(issuer: str) -> dict | None: xrp = 0.0 return {"xrp_liquidity": xrp, "trading_fee_bps": amm.get("trading_fee")} except Exception as exc: - logger.warning("flow: xrpl amm pulse failed on %s: %s", node, type(exc).__name__) + logger.warning( + "flow: xrpl amm pulse failed on %s: %s", node, type(exc).__name__ + ) continue return None # ── synthesis (pure, importable) ─────────────────────────────────────────── + def synthesize(global_d, markets_d, trending_syms, solana_pulse, xrpl_amm) -> dict: """Turn raw fetches into a multi-asset flow composite. Returns a structured dict.""" # --- risk regime from /global --- - regime = {"label": "NEUTRAL", "mcap_change_24h": 0.0, "btc_dominance": 0.0, - "eth_dominance": 0.0, "score": 0.0} + regime = { + "label": "NEUTRAL", + "mcap_change_24h": 0.0, + "btc_dominance": 0.0, + "eth_dominance": 0.0, + "score": 0.0, + } if global_d: g = global_d.get("data", {}) mc = _num(g.get("market_cap_change_percentage_24h_usd")) @@ -208,12 +248,16 @@ def synthesize(global_d, markets_d, trending_syms, solana_pulse, xrpl_amm) -> di regime["eth_dominance"] = round(eth_d, 2) risk_score = _clamp(mc / 5.0) - _clamp((btc_d - 50.0) / 10.0) * 0.3 regime["score"] = round(_clamp(risk_score), 3) - regime["label"] = "RISK-ON" if regime["score"] > 0.15 else ("RISK-OFF" if regime["score"] < -0.15 else "NEUTRAL") + regime["label"] = ( + "RISK-ON" + if regime["score"] > 0.15 + else ("RISK-OFF" if regime["score"] < -0.15 else "NEUTRAL") + ) # --- per-asset flow intensity from /coins/markets --- trending_set = {s.upper() for s in trending_syms} assets_out = [] - for m in (markets_d or []): + for m in markets_d or []: sym = (m.get("symbol") or "").upper() mcap = _num(m.get("market_cap")) vol = _num(m.get("total_volume")) @@ -226,14 +270,16 @@ def synthesize(global_d, markets_d, trending_syms, solana_pulse, xrpl_amm) -> di flow = _clamp(vol_mcap * 5.0, -0.5, 0.5) * 0.4 + _clamp(chg / 6.0) * 0.6 if sym in trending_set: flow = _clamp(flow + 0.1) - assets_out.append({ - "symbol": sym, - "pair": f"{sym}-USDC", - "flow_score": round(flow, 3), - "volume_to_mcap": round(vol_mcap, 3), - "price_change_24h": round(chg, 2), - "trending": sym in trending_set, - }) + assets_out.append( + { + "symbol": sym, + "pair": f"{sym}-USDC", + "flow_score": round(flow, 3), + "volume_to_mcap": round(vol_mcap, 3), + "price_change_24h": round(chg, 2), + "trending": sym in trending_set, + } + ) assets_out.sort(key=lambda a: abs(a["flow_score"]), reverse=True) # --- on-chain pulse (Solana default; XRPL optional) --- @@ -276,25 +322,34 @@ def _format_verdict(sig: dict) -> str: f"- Verdict: **{sig['direction']}** ({sig['rationale']})", ] if best: - lines.append(f"- Best flow: **{best['symbol']}** score {best['flow_score']:+.2f} " - f"(vol/mcap {best['volume_to_mcap']}, 24h {best['price_change_24h']:+.2f}%, trend={best['trending']})") + lines.append( + f"- Best flow: **{best['symbol']}** score {best['flow_score']:+.2f} " + f"(vol/mcap {best['volume_to_mcap']}, 24h {best['price_change_24h']:+.2f}%, trend={best['trending']})" + ) sp = sig["onchain"].get("solana") if sp: - lines.append(f"- Solana on-chain pulse: flow {sp['flow_score']:+.2f}, " - f"top-pool vol24h ${sp['total_vol24h']:,.0f}, median chg {sp['median_chg24h']:+.1f}%") + lines.append( + f"- Solana on-chain pulse: flow {sp['flow_score']:+.2f}, " + f"top-pool vol24h ${sp['total_vol24h']:,.0f}, median chg {sp['median_chg24h']:+.1f}%" + ) for p in sp["pools"][:4]: - lines.append(f" - {p['name'][:24]:24} vol ${p['vol24h']:,.0f} chg {p['chg24h']:+.1f}%") + lines.append( + f" - {p['name'][:24]:24} vol ${p['vol24h']:,.0f} chg {p['chg24h']:+.1f}%" + ) if sig["onchain"].get("xrpl"): lines.append(f"- XRPL AMM pulse: {sig['onchain']['xrpl']}") lines.append("- Per-asset:") for a in sig["assets"][:5]: - lines.append(f" - {a['symbol']}: {a['flow_score']:+.2f} " - f"(vol/mcap {a['volume_to_mcap']}, 24h {a['price_change_24h']:+.2f}%, trend={a['trending']})") + lines.append( + f" - {a['symbol']}: {a['flow_score']:+.2f} " + f"(vol/mcap {a['volume_to_mcap']}, 24h {a['price_change_24h']:+.2f}%, trend={a['trending']})" + ) return "\n".join(lines) # ── orchestration ────────────────────────────────────────────────────────── + async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: """Entry point invoked by manage_routines. Fetches, scores, reports, returns.""" ids = [s.strip() for s in config.context_assets.split(",") if s.strip()] @@ -305,7 +360,11 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: fetch_markets(ids), fetch_solana_pulse(config.solana_top_n), ) - xrpl_amm = await fetch_xrpl_amm_pulse(config.xrpl_amm_issuer) if config.xrpl_amm_issuer else None + xrpl_amm = ( + await fetch_xrpl_amm_pulse(config.xrpl_amm_issuer) + if config.xrpl_amm_issuer + else None + ) sig = synthesize(global_d, markets, trending, solana, xrpl_amm) @@ -314,8 +373,13 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: from condor.reports import ReportBuilder builder = ReportBuilder("Smart-Money Flow") - builder.source("routine", "onchain_flow").tags(["flow", "on-chain", "solana", "smart-money"]) - builder.section("01 / RISK REGIME", "Cross-market risk-on/off from total mcap momentum and BTC dominance") + builder.source("routine", "onchain_flow").tags( + ["flow", "on-chain", "solana", "smart-money"] + ) + builder.section( + "01 / RISK REGIME", + "Cross-market risk-on/off from total mcap momentum and BTC dominance", + ) builder.kpi("Regime", sig["regime"]["label"]) builder.kpi("Mcap 24h", f"{sig['regime']['mcap_change_24h']:+.2f}%") builder.kpi("BTC Dominance", f"{sig['regime']['btc_dominance']:.1f}%") @@ -323,14 +387,29 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: builder.kpi("Solana Flow", f"{sig['solana_flow']:+.2f}") sp = sig["onchain"].get("solana") if sp: - builder.section("02 / SOLANA ON-CHAIN PULSE", "SOL top pools by 24h volume — crypto-native flow") + builder.section( + "02 / SOLANA ON-CHAIN PULSE", + "SOL top pools by 24h volume — crypto-native flow", + ) builder.kpi("Top-pool Vol 24h", f"${sp['total_vol24h']:,.0f}") builder.kpi("Median Chg 24h", f"{sp['median_chg24h']:+.1f}%") builder.kpi("On-chain Flow", f"{sp['flow_score']:+.2f}") builder.table(sp["pools"][:8], ["name", "dex", "vol24h", "chg24h", "tvl"]) if sig["assets"]: - builder.section("03 / CROSS-MARKET CONTEXT", "Market-wide flow intensity (BTC/ETH/SOL)") - builder.table(sig["assets"][:6], ["symbol", "pair", "flow_score", "volume_to_mcap", "price_change_24h", "trending"]) + builder.section( + "03 / CROSS-MARKET CONTEXT", "Market-wide flow intensity (BTC/ETH/SOL)" + ) + builder.table( + sig["assets"][:6], + [ + "symbol", + "pair", + "flow_score", + "volume_to_mcap", + "price_change_24h", + "trending", + ], + ) builder.markdown(_format_verdict(sig)) builder.manual_order() await builder.save() diff --git a/agents/solana_dex_lp_expert/AGENT.md b/agents/solana_dex_lp_expert/AGENT.md index d55aec027..1f7a2d857 100644 --- a/agents/solana_dex_lp_expert/AGENT.md +++ b/agents/solana_dex_lp_expert/AGENT.md @@ -11,6 +11,9 @@ tools: - get_portfolio_overview - get_market_data - search_history +- manage_routines +- manage_trading_agent +- trading_agent_journal_read - manage_memory - manage_skill when_to_consult: When the user asks which Solana memecoin pools to LP now, how to rank diff --git a/agents/solana_dex_lp_expert/routines/lp_scanner.py b/agents/solana_dex_lp_expert/routines/lp_scanner.py index f374459fd..912749d45 100644 --- a/agents/solana_dex_lp_expert/routines/lp_scanner.py +++ b/agents/solana_dex_lp_expert/routines/lp_scanner.py @@ -16,12 +16,13 @@ hold — the agent should never open a 2nd slot on the same pool or token. """ -import logging import asyncio +import logging import re -import aiohttp + from pydantic import BaseModel, Field, field_validator from telegram.ext import ContextTypes + from config_manager import get_client from handlers.dex.geckoterminal import _extract_pool_data @@ -29,7 +30,6 @@ CATEGORY = "Analysis" -GECKO_BASE = "https://api.geckoterminal.com/api/v2" GECKO_NETWORK = "solana" # GeckoTerminal network id CLMM_NETWORK = "solana-mainnet-beta" # hummingbot-api CLMM network id _WINDOW_TO_FIELD = {"1h": "volume_1h", "6h": "volume_6h", "24h": "volume_24h"} @@ -71,17 +71,16 @@ def _coerce_list(cls, v): return v -async def _gecko_get( - session: aiohttp.ClientSession, path: str, params: dict | None = None -) -> dict: - url = f"{GECKO_BASE}/{path}" - headers = {"Accept": "application/json;version=20230302"} - async with session.get( - url, headers=headers, params=params, timeout=aiohttp.ClientTimeout(total=25) - ) as resp: - if resp.status != 200: - raise RuntimeError(f"GeckoTerminal {path} -> HTTP {resp.status}") - return await resp.json() +async def _gecko_get(path: str, params: dict | None = None) -> dict: + """One GeckoTerminal call on the process-wide budget. + + Goes through ``pool_data`` rather than opening a session here: gecko's free + tier is ~30 requests/minute per IP, shared with the dashboard and every other + routine, and a scanner that fans out per venue is exactly what exhausts it. + """ + from condor.pool_data import gecko_request + + return await gecko_request("GET", path, params=params) def _num(v, default=0.0) -> float: @@ -108,17 +107,12 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: # 1. Source candidates from GeckoTerminal: global trending + top-per-venue. raw: list[dict] = [] try: - async with aiohttp.ClientSession() as session: - tasks = [_gecko_get(session, f"networks/{GECKO_NETWORK}/trending_pools")] - for v in venues: - tasks.append( - _gecko_get( - session, - f"networks/{GECKO_NETWORK}/dexes/{v}/pools", - {"page": 1}, - ) - ) - results = await asyncio.gather(*tasks, return_exceptions=True) + tasks = [_gecko_get(f"networks/{GECKO_NETWORK}/trending_pools")] + for v in venues: + tasks.append( + _gecko_get(f"networks/{GECKO_NETWORK}/dexes/{v}/pools", {"page": 1}) + ) + results = await asyncio.gather(*tasks, return_exceptions=True) for r in results: if isinstance(r, Exception): logger.warning(f"lp_scanner: gecko source failed: {r}") diff --git a/agents/xrpl_market_maker/AGENT.md b/agents/xrpl_market_maker/AGENT.md index 2a3654001..8e51c5c94 100644 --- a/agents/xrpl_market_maker/AGENT.md +++ b/agents/xrpl_market_maker/AGENT.md @@ -11,6 +11,8 @@ tools: - manage_controllers - manage_bots - manage_routines +- manage_trading_agent +- trading_agent_journal_read - manage_skill - send_notification when_to_consult: When the user asks about quoting on the XRP Ledger DEX — whether a diff --git a/agents/xrpl_market_maker/routines/xrpl_mm_quote_planner.py b/agents/xrpl_market_maker/routines/xrpl_mm_quote_planner.py index 8a5d76435..ce497905f 100644 --- a/agents/xrpl_market_maker/routines/xrpl_mm_quote_planner.py +++ b/agents/xrpl_market_maker/routines/xrpl_mm_quote_planner.py @@ -1,4 +1,5 @@ """Plan XRPL CLOB maker quotes: reference fair value, spread bounds, viability verdict.""" + import asyncio import logging import math @@ -27,10 +28,30 @@ # 733 days of Bitget perp 1H bars. Values are vol relative to the daily average. # Trough 04:00-11:00 UTC (Asia afternoon / pre-Europe); peak 13:00-15:00 UTC (US open). HOUR_VOL_MULT = { - 0: 1.03, 1: 1.10, 2: 0.95, 3: 0.89, 4: 0.82, 5: 0.85, - 6: 0.82, 7: 0.82, 8: 0.91, 9: 0.83, 10: 0.78, 11: 0.82, - 12: 0.92, 13: 1.28, 14: 1.50, 15: 1.40, 16: 1.16, 17: 1.21, - 18: 1.07, 19: 1.04, 20: 1.02, 21: 0.97, 22: 0.97, 23: 0.82, + 0: 1.03, + 1: 1.10, + 2: 0.95, + 3: 0.89, + 4: 0.82, + 5: 0.85, + 6: 0.82, + 7: 0.82, + 8: 0.91, + 9: 0.83, + 10: 0.78, + 11: 0.82, + 12: 0.92, + 13: 1.28, + 14: 1.50, + 15: 1.40, + 16: 1.16, + 17: 1.21, + 18: 1.07, + 19: 1.04, + 20: 1.02, + 21: 0.97, + 22: 0.97, + 23: 0.82, } MS_PER_HOUR = 3_600_000 @@ -38,11 +59,15 @@ class Config(BaseModel): """Plan maker quotes for an XRPL CLOB pair against a CEX reference price.""" - xrpl_pair: str = Field(default="RLUSD-XRP", description="Pair as the xrpl connector names it") + xrpl_pair: str = Field( + default="RLUSD-XRP", description="Pair as the xrpl connector names it" + ) reference_connector: str = Field( default="bitget_perpetual", description="CEX connector supplying fair value" ) - reference_pair: str = Field(default="XRP-USDT", description="Reference pair for XRP/USD") + reference_pair: str = Field( + default="XRP-USDT", description="Reference pair for XRP/USD" + ) tick_interval_sec: int = Field(default=300, description="Seconds between LLM ticks") requote_interval_sec: int = Field( default=0, @@ -51,9 +76,12 @@ class Config(BaseModel): "tick — the bot requotes without the agent, so the LLM tick must not set the floor", ) levels_per_side: int = Field(default=3, description="Quote levels per side") - total_amount_quote: float = Field(default=100.0, description="Capital to deploy, USD") + total_amount_quote: float = Field( + default=100.0, description="Capital to deploy, USD" + ) adverse_k: float = Field( - default=1.0, description="Multiplier on expected adverse move; higher = wider floor" + default=1.0, + description="Multiplier on expected adverse move; higher = wider floor", ) use_vol_clock: bool = Field( default=True, @@ -63,9 +91,12 @@ class Config(BaseModel): default=0.1, description="Assumed AMM fee %% if amm_info is unreachable" ) amm_asset2_issuer: str = Field( - default="", description="Issuer address of the non-XRP asset (blank = skip amm_info)" + default="", + description="Issuer address of the non-XRP asset (blank = skip amm_info)", + ) + amm_asset2_currency: str = Field( + default="RLUSD", description="Non-XRP asset currency code" ) - amm_asset2_currency: str = Field(default="RLUSD", description="Non-XRP asset currency code") # ── helpers ────────────────────────────────────────────────────────────────── @@ -115,7 +146,12 @@ async def _fetch_amm_fee_pct(currency: str, issuer: str) -> tuple[float | None, return None, "no issuer configured — amm_info skipped" payload = { "method": "amm_info", - "params": [{"asset": {"currency": "XRP"}, "asset2": {"currency": currency, "issuer": issuer}}], + "params": [ + { + "asset": {"currency": "XRP"}, + "asset2": {"currency": currency, "issuer": issuer}, + } + ], } try: async with httpx.AsyncClient(timeout=10) as http: @@ -180,9 +216,14 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: try: candles_raw, ref_prices = await asyncio.gather( client.market_data.get_candles( - config.reference_connector, config.reference_pair, interval="1m", max_records=120 + config.reference_connector, + config.reference_pair, + interval="1m", + max_records=120, + ), + client.market_data.get_prices( + config.reference_connector, [config.reference_pair] ), - client.market_data.get_prices(config.reference_connector, [config.reference_pair]), ) except Exception as exc: return f"reference_price: ERROR fetching {config.reference_pair} — {exc}" @@ -198,7 +239,9 @@ async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: implied_xrpl_price = (1.0 / ref_mid) if ref_mid > 0 else 0.0 out.append("=== REFERENCE (fair value source) ===") - out.append(f"reference_pair: {config.reference_pair} @ {config.reference_connector}") + out.append( + f"reference_pair: {config.reference_pair} @ {config.reference_connector}" + ) out.append(f"reference_mid_usd: {ref_mid:.6f}") out.append(f"implied_{config.xrpl_pair}: {implied_xrpl_price:.6f} (1 / XRP-USD)") @@ -251,7 +294,9 @@ def _utc(ms: int) -> str: f"(UTC {_utc(now_ms)}-{_utc(fwd_end_ms)}, quote live window)" ) out.append(f"vol_adjusted: {vol_adj * math.sqrt(60) * 100:.4f}% per minute") - out.append(f"tick_interval_sec: {config.tick_interval_sec} (LLM reasoning cadence)") + out.append( + f"tick_interval_sec: {config.tick_interval_sec} (LLM reasoning cadence)" + ) out.append( f"requote_interval_sec: {requote_sec} (quote exposure — THIS sets the floor)" ) @@ -281,7 +326,9 @@ def _utc(ms: int) -> str: out.append("") out.append("=== SPREAD CEILING (AMM fee to undercut) ===") out.append(f"amm_trading_fee_pct: {amm_fee_pct:.4f}% ({fee_note})") - out.append(f"spread_ceiling_bps: {ceiling_bps:.2f} (quote wider and flow routes to the AMM)") + out.append( + f"spread_ceiling_bps: {ceiling_bps:.2f} (quote wider and flow routes to the AMM)" + ) # 4. Viability — the load-bearing verdict. viable = floor_bps < ceiling_bps @@ -345,7 +392,9 @@ def _utc(ms: int) -> str: # A controller's total_amount_quote is denominated in the pair's QUOTE asset. On # RLUSD-XRP that is XRP, not USD — passing a USD figure straight through oversizes # the deployment by the XRP price (~3x), silently breaching the risk limit. - quote_asset = config.xrpl_pair.split("-")[-1].upper() if "-" in config.xrpl_pair else "" + quote_asset = ( + config.xrpl_pair.split("-")[-1].upper() if "-" in config.xrpl_pair else "" + ) if quote_asset == "XRP" and ref_mid > 0: amount_in_quote = config.total_amount_quote / ref_mid out.append( @@ -375,8 +424,12 @@ def _utc(ms: int) -> str: bids = (book or {}).get("bids") or [] asks = (book or {}).get("asks") or [] if bids and asks: - best_bid = float(bids[0][0] if isinstance(bids[0], (list, tuple)) else bids[0]["price"]) - best_ask = float(asks[0][0] if isinstance(asks[0], (list, tuple)) else asks[0]["price"]) + best_bid = float( + bids[0][0] if isinstance(bids[0], (list, tuple)) else bids[0]["price"] + ) + best_ask = float( + asks[0][0] if isinstance(asks[0], (list, tuple)) else asks[0]["price"] + ) mid = (best_bid + best_ask) / 2 out.append(f"best_bid: {best_bid:.6f}") out.append(f"best_ask: {best_ask:.6f}") @@ -391,6 +444,8 @@ def _utc(ms: int) -> str: out.append("status: EMPTY OR UNAVAILABLE — do not quote blind") except Exception as exc: out.append(f"status: ERROR — {exc}") - out.append("action: treat as a hard stop; do not place offers without live book state") + out.append( + "action: treat as a hard stop; do not place offers without live book state" + ) return "\n".join(out) diff --git a/condor/acp/__init__.py b/condor/acp/__init__.py index 65c3804af..ffe6962d4 100644 --- a/condor/acp/__init__.py +++ b/condor/acp/__init__.py @@ -1,14 +1,14 @@ from .client import ( - ACPClient, ACP_COMMANDS, - resolve_acp, + ACPClient, + ACPEvent, + Heartbeat, PermissionCallback, + PromptDone, TextChunk, ThoughtChunk, ToolCallEvent, ToolCallUpdate, - PromptDone, - Heartbeat, - ACPEvent, + resolve_acp, ) from .pydantic_ai_client import PydanticAIClient, is_pydantic_ai_model diff --git a/condor/acp/jsonrpc.py b/condor/acp/jsonrpc.py index 5e9e0e5a4..c1563553d 100644 --- a/condor/acp/jsonrpc.py +++ b/condor/acp/jsonrpc.py @@ -23,7 +23,9 @@ def __init__(self, code: int, message: str, data: Any = None): detail = str(data.get("details") or data.get("message") or "") or str(data) elif data: detail = str(data) - super().__init__(f"[{code}] {message}" + (f": {detail[:300]}" if detail else "")) + super().__init__( + f"[{code}] {message}" + (f": {detail[:300]}" if detail else "") + ) # Standard JSON-RPC 2.0 error codes @@ -91,7 +93,9 @@ async def handle_line(self, line: str, writer: asyncio.StreamWriter) -> None: if "error" in data: err = data["error"] future.set_exception( - JSONRPCError(err.get("code", -1), err.get("message", ""), err.get("data")) + JSONRPCError( + err.get("code", -1), err.get("message", ""), err.get("data") + ) ) else: future.set_result(data.get("result")) @@ -108,7 +112,10 @@ async def handle_line(self, line: str, writer: asyncio.StreamWriter) -> None: if msg_id is not None: resp = { "jsonrpc": "2.0", - "error": {"code": METHOD_NOT_FOUND, "message": f"Method not found: {method}"}, + "error": { + "code": METHOD_NOT_FOUND, + "message": f"Method not found: {method}", + }, "id": msg_id, } writer.write((json.dumps(resp) + "\n").encode()) @@ -116,7 +123,11 @@ async def handle_line(self, line: str, writer: asyncio.StreamWriter) -> None: return try: - result = handler(**params) if not asyncio.iscoroutinefunction(handler) else await handler(**params) + result = ( + handler(**params) + if not asyncio.iscoroutinefunction(handler) + else await handler(**params) + ) except Exception as e: log.exception("Handler error for %s", method) if msg_id is not None: diff --git a/condor/agents/README.md b/condor/agents/README.md index f140e4e9f..33e7ca69e 100644 --- a/condor/agents/README.md +++ b/condor/agents/README.md @@ -63,7 +63,7 @@ In short: executors give us the cleanest possible boundary between "what this ag | `engine.py` | `TickEngine` — one instance per running agent. Runs the tick loop, builds prompts, drives ACP sessions, captures tool calls, persists snapshots. | | `strategy.py` | `Strategy` + `StrategyStore`. Strategies live as `agent.md` files (YAML frontmatter + markdown body) under `agents/{slug}/`. | | `journal.py` | `JournalManager` — compact, human-readable per-session memory: summary, decisions, ticks, snapshots, executors. Also `learnings.md` cross-session. | -| `prompts.py` | Builds the per-tick prompt: system prompt + strategy + provider summaries + journal context + risk state + user directives. | +| `prompts.py` | Builds the per-tick prompt: system prompt + strategy + provider summaries + journal context + risk state. | | `risk.py` | `RiskEngine` + `RiskLimits`. Hard guardrails (max exposure, max drawdown, max open executors) and the permission callback that auto-approves tool calls only if they pass the risk check. | | `providers/` | Deterministic pre-tick data fetchers. Two core providers ship: `executors.py` (filtered by `controller_id`) and `positions.py` (positions summary, also filtered by `controller_id`). | | `config.py` | `AgentConfig` schema persisted per session. | @@ -94,7 +94,7 @@ agents/ 2. **Run core providers** — currently `executors` and `positions`, both filtered by `controller_id == agent_id`. Their structured `data` is kept for tracking; their `summary` strings go into the prompt. 3. **Read journal context** — `learnings.md`, `summary`, and the last 3 decisions. 4. **Get risk state** from `RiskEngine` (exposure, drawdown, open count). If the agent is blocked, log it and return without invoking the LLM. -5. **Build the prompt** via `build_tick_prompt(...)`, optionally appending any user directives queued via `inject_directive()`. +5. **Build the prompt** via `build_tick_prompt(...)`. 6. **Spawn an ACP session** for the strategy's `agent_key` (claude-code, gemini, copilot…) with MCP servers wired in (Hummingbot tools, market data, notifications, journal writes). Stream events, capture text and tool calls. 7. **Persist** — for sessions, write `snapshot_N.md`, append a tick to the journal, and update the running summary. For dry-runs / run-once, write a flat `experiment_N.md`. @@ -258,14 +258,6 @@ In production, agents are normally started/stopped from the Telegram `/agent` fl - `agents/{slug}/learnings.md` — lessons the agent has chosen to keep across sessions. - `agents/{slug}/dry_runs/experiment_N.md` — dry-run / run-once results. -### 4.5 Injecting directives mid-flight - -```python -engine.inject_directive("Reduce exposure, news event in 10 min.") -``` - -The next tick's prompt will include the directive verbatim under a `USER DIRECTIVES` section, then clear it. - --- ## 5. Mental model diff --git a/condor/agents/canvas.py b/condor/agents/canvas.py index 1256134ff..2a4f7cacf 100644 --- a/condor/agents/canvas.py +++ b/condor/agents/canvas.py @@ -18,12 +18,12 @@ from __future__ import annotations -import os import re -import tempfile from datetime import datetime, timezone from pathlib import Path +from condor.fsutil import atomic_write_text + CANVAS_FILE = "canvas.md" REVISIONS_FILE = "canvas_revisions.md" @@ -211,31 +211,12 @@ def _append_revision( def _atomic_write(path: Path, content: str) -> None: - """tmpfile + ``os.replace``, matching ``condor/reports/store.py``. + """tmpfile + ``os.replace``, via the shared :mod:`condor.fsutil` helper. A tick can be cancelled mid-write; a half-written canvas would then be inlined into the next prompt. """ - path.parent.mkdir(parents=True, exist_ok=True) - tmp = tempfile.NamedTemporaryFile( - mode="w", - encoding="utf-8", - dir=str(path.parent), - prefix=f".{path.name}.", - suffix=".tmp", - delete=False, - ) - try: - tmp.write(content) - tmp.close() - os.replace(tmp.name, path) - except Exception: - tmp.close() - try: - os.unlink(tmp.name) - except OSError: - pass - raise + atomic_write_text(path, content) # --------------------------------------------------------------------------- diff --git a/condor/agents/config.py b/condor/agents/config.py index f7d536aa6..88b418acd 100644 --- a/condor/agents/config.py +++ b/condor/agents/config.py @@ -44,6 +44,11 @@ class AgentConfig(BaseModel): description="Total capital budget for this session in quote currency", ) frequency_sec: int = Field(default=60, description="Tick frequency in seconds") + tick_timeout_sec: int = Field( + default=0, + description="Wall-clock budget for one tick's agent session, in seconds; " + "0 = use the runtime default (10 min, or CONDOR_TIMEOUT_TICK_DEFAULT)", + ) trading_context: str = Field( default="", description="Natural language session context that guides the agent's trading decisions", diff --git a/condor/agents/delegate.py b/condor/agents/delegate.py index f4ee1f36e..618e90245 100644 --- a/condor/agents/delegate.py +++ b/condor/agents/delegate.py @@ -20,6 +20,15 @@ when the task starts, so a delegation killed by a restart is reported as ``interrupted`` instead of vanishing without a trace. It is never auto-restarted: delegations are one-shot and re-running could duplicate side effects. + +Ephemeral does not mean free: the registry is bounded on both axes since +CORR-143. A finished delegation is retired once its transcript, events sidecar +and status record are on disk, and only the most recent +:data:`MAX_FINISHED_DELEGATIONS` of them stay resident -- older ones are served +from :mod:`condor.agents.delegation_history` instead. Within one delegation the +event stream is bounded too (:data:`MAX_EVENTS_PER_DELEGATION`, with tool +payloads clipped to what a reader would have been shown anyway), so no single +runaway session can pin an unbounded transcript. """ from __future__ import annotations @@ -39,10 +48,47 @@ # Default per-task wall-clock budget; a hung ACP subprocess is cancelled after this. DEFAULT_TIMEOUT_S = 900 -# Ceiling on a single tool output wherever a transcript is *read* -- the on-disk +# Ceiling on a single tool payload wherever a transcript is *read* -- the on-disk # markdown and the wire projection share it so the two can never disagree about -# where a huge output was cut. ``dt.events`` itself keeps the full output. +# where a huge output was cut. Since CORR-143 the event sink applies the same +# ceiling as it folds, so what a delegation keeps in RAM is what every reader +# would have shown anyway; :func:`_clip_output` is idempotent precisely so that +# clipping once at write time and again at read time yields the same string. MAX_TOOL_OUTPUT = 2000 +TRUNCATION_MARKER = "\n… (truncated)" + +# How many events one delegation retains. The event stream is the heavy half of +# a DelegateTask, so it is bounded per task and not only across tasks: past this +# the sink drops the oldest events. Generous on purpose -- an ordinary +# delegation never reaches it; what it stops is a runaway session pinning an +# unbounded transcript in a process designed to run for weeks. +MAX_EVENTS_PER_DELEGATION = 500 + +# How much of one merged thought/text run stays in a single event. Consecutive +# chunks are folded into the last entry, so without a roll-over a chatty agent +# grows one string without bound -- which no count of events can cap. Past this +# the sink opens a new entry, subject to the event bound like every other one. +MAX_CHUNK_CHARS = 4000 + +# The in-memory record of a cut: the sink drops the oldest events past the bound +# and folds how many into this marker, so a transcript never silently claims to +# be complete. It reaches readers as a plain text note (``events_for_wire``), so +# no client has to learn a new event type. +DROPPED_EVENT_TYPE = "dropped" + +# How many *terminal* delegations stay resident (CORR-143). Deliberately a count +# bound and not a TTL: the MCP ``delegate`` contract is submit-now/collect-later +# with no deadline, so there is no grace window that is the right one to expire +# on. What makes a late collect safe is the disk fallback, not a timer -- +# ``GET /agents/delegations/{task_id}`` and its ``/events`` sibling both fall +# through to :mod:`condor.agents.delegation_history`, which rebuilds the record +# from the ``{task_id}.status.json`` and ``{task_id}.events.json`` files written +# in ``_run``'s finally block, i.e. before an entry can ever be evicted. The two +# registry-only readers (``GET /agents/delegations`` and the ``/delegations`` +# Telegram list) are per-process snapshots by contract and have the merged +# ``/delegations/history`` route as their complete view. In-flight delegations +# are never evicted, at any count. +MAX_FINISHED_DELEGATIONS = 25 # What happens to the conversation that asked for the work when the task ends. # @@ -232,22 +278,35 @@ def _make_event_sink(dt: DelegateTask): tl = dt.events tc_map: dict[str, dict] = {} + def append(entry: dict) -> None: + tl.append(entry) + _bound_events(tl) + + def merge_chunk(kind: str, text: str) -> None: + # Roll into a new entry once the current run is long enough: merging for + # ever would grow one string without bound, which no count of events can + # cap (CORR-143). + if tl and tl[-1]["type"] == kind and len(tl[-1]["text"]) < MAX_CHUNK_CHARS: + tl[-1]["text"] += text + else: + append({"type": kind, "text": text}) + def sink(event) -> None: if isinstance(event, ThoughtChunk): - if tl and tl[-1]["type"] == "thought": - tl[-1]["text"] += event.text - else: - tl.append({"type": "thought", "text": event.text}) + merge_chunk("thought", event.text) elif isinstance(event, TextChunk): - if tl and tl[-1]["type"] == "text": - tl[-1]["text"] += event.text - else: - tl.append({"type": "text", "text": event.text}) + merge_chunk("text", event.text) elif isinstance(event, (ToolCallEvent, ToolCallUpdate)): tc = fold_tool_call_event(tc_map, event) if tc is not None: tc["type"] = "tool" - tl.append(tc) + append(tc) + # An output arrives on a *patch*, which the fold applies in place and + # reports as ``None`` -- so the payload bound is applied to the + # folded entry looked up here, not to the return value. + folded = tc_map.get(event.tool_call_id) + if folded is not None: + _bound_tool_payloads(folded) return sink @@ -302,6 +361,10 @@ async def _run(dt: DelegateTask, bot, timeout_s: int) -> None: # already in the chat and in the transcript. if dt.on_complete == ON_COMPLETE_RESUME and dt.status == "done": await _resume_conversation(dt) + # Last of all, and only once the transcript, the sidecar and the status + # record are on disk: this may drop older finished entries, and nothing + # may be evicted that a reader cannot still find in history. + retire_delegation(dt) async def stop_delegation(task_id: str) -> bool: @@ -315,11 +378,88 @@ async def stop_delegation(task_id: str) -> bool: def _clip_output(value) -> str: - """Stringify a tool output, bounded at :data:`MAX_TOOL_OUTPUT`.""" + """Stringify a tool payload, bounded at :data:`MAX_TOOL_OUTPUT`. + + Idempotent: an already-clipped string comes back unchanged, so the sink can + apply the bound as the stream arrives (CORR-143) and the two readers can + keep applying it on the way out without stacking a second marker. + """ out = str(value) - if len(out) > MAX_TOOL_OUTPUT: - return out[:MAX_TOOL_OUTPUT] + "\n… (truncated)" - return out + if out.endswith(TRUNCATION_MARKER) or len(out) <= MAX_TOOL_OUTPUT: + return out + return out[:MAX_TOOL_OUTPUT] + TRUNCATION_MARKER + + +def _bound_tool_payloads(tc: dict) -> None: + """Clip a folded tool entry's payloads to what a reader would have shown. + + The output is what gets big (a market-data dump, a file read), and both + projections already cut it at :data:`MAX_TOOL_OUTPUT` -- keeping the full + string in RAM bought nothing. The input is normally small and is left in its + original shape so the dashboard can render it as JSON; only one that + serializes past the same ceiling degrades to its clipped string form, since + an ``Edit``/``Write`` call can carry a whole file in its arguments. + """ + import json + + out = tc.get("output") + if out is not None: + tc["output"] = _clip_output(out) + inp = tc.get("input") + if inp is not None and not isinstance(inp, str): + serialized = json.dumps(inp, default=str) + if len(serialized) > MAX_TOOL_OUTPUT: + tc["input"] = _clip_output(serialized) + + +def _bound_events(events: list[dict]) -> None: + """Keep an event stream under :data:`MAX_EVENTS_PER_DELEGATION`, oldest first. + + The head is what gets dropped: a reader of a runaway session cares about how + it ended, and the tail is also what the completion notice is drawn from. The + count of dropped events is folded into a leading marker so the cut is + visible in the transcript rather than silent. + """ + if len(events) <= MAX_EVENTS_PER_DELEGATION: + return + if not events or events[0].get("type") != DROPPED_EVENT_TYPE: + events.insert(0, {"type": DROPPED_EVENT_TYPE, "count": 0}) + marker = events[0] + while len(events) > MAX_EVENTS_PER_DELEGATION: + events.pop(1) # index 0 is the marker + marker["count"] += 1 + + +def _dropped_note(count: int) -> str: + return f"_… {count} earlier event(s) dropped to bound memory (CORR-143)_" + + +def retire_delegation(dt: "DelegateTask") -> int: + """Move a finished delegation to the young end and evict the oldest ones. + + Called once ``_run`` has persisted the transcript, the events sidecar and + the status record, so everything evicted here is already readable from disk. + Re-inserting refreshes recency (dicts keep insertion order), which makes the + registry's tail *finish* order rather than start order -- a slow delegation + started first must not be the first evicted when it finally lands. + + Returns how many entries were evicted. Public so the same retirement can be + exercised directly rather than only through a live task. + """ + if _delegations.get(dt.task_id) is dt: + # Never re-register a task that was already dropped: the pop/insert pair + # would resurrect it as the newest entry. + _delegations.pop(dt.task_id) + _delegations[dt.task_id] = dt + + finished = [t for t, d in _delegations.items() if d.status in TERMINAL_STATES] + evicted = 0 + for task_id in finished[: max(len(finished) - MAX_FINISHED_DELEGATIONS, 0)]: + _delegations.pop(task_id, None) + evicted += 1 + if evicted: + log.debug("Evicted %s finished delegation(s) from the registry", evicted) + return evicted def events_for_wire(events: list[dict]) -> list[dict]: @@ -338,6 +478,10 @@ def events_for_wire(events: list[dict]) -> list[dict]: kind = ev.get("type") if kind in ("thought", "text"): wire.append({"type": kind, "text": ev.get("text") or ""}) + elif kind == DROPPED_EVENT_TYPE: + # Projected as a text note rather than a new wire type: every client + # already renders text, and a cut nobody renders is a silent lie. + wire.append({"type": "text", "text": _dropped_note(ev.get("count") or 0)}) elif kind == "tool": inp = ev.get("input") out = ev.get("output") @@ -367,7 +511,9 @@ def _render_session(events: list[dict]) -> str: tool_n = 0 for ev in events: kind = ev.get("type") - if kind == "thought": + if kind == DROPPED_EVENT_TYPE: + parts.append(_dropped_note(ev.get("count") or 0)) + elif kind == "thought": text = (ev.get("text") or "").strip() if text: quoted = "\n".join(f"> {line}" for line in text.splitlines()) diff --git a/condor/agents/engine.py b/condor/agents/engine.py index 820cdbe42..5be524798 100644 --- a/condor/agents/engine.py +++ b/condor/agents/engine.py @@ -28,6 +28,8 @@ ) from condor.acp.pydantic_ai_client import PydanticAIClient, is_pydantic_ai_model from condor.runtime.registry_file import LoopState +from condor.runtime.timeouts import resolve_tick_timeout +from condor.telemetry import taps as telemetry_taps from .agent import Agent from .journal import JournalManager, next_experiment_number, next_session_number @@ -106,10 +108,12 @@ class TickEngine: _last_tick_at: float = field(default=0.0, init=False) _last_error: str = field(default="", init=False) _last_skill_data: dict[str, Any] = field(default_factory=dict, init=False) - _pending_directives: list[str] = field(default_factory=list, init=False) _cached_routines_section: str | None = field(default=None, init=False, repr=False) _adoption_done: bool = field(default=False, init=False, repr=False) _mode_mismatch_noted: bool = field(default=False, init=False, repr=False) + # Why the loop ended, for the strategy_run telemetry event: "user" unless + # something in the loop set it first. + _last_stop_reason: str = field(default="user", init=False, repr=False) # Session canvas + live report (FEAT-036). Both None for experiments, which # keep no journal and therefore no narrative to render. _session_report: "SessionReport | None" = field( @@ -168,8 +172,6 @@ def __post_init__(self): self.journal = JournalManager( self.agent_id, - strategy_name=self.strategy.name, - strategy_description=self.strategy.description, session_dir=self.session_dir, agent_dir=strategy_dir, ) @@ -261,6 +263,13 @@ async def stop(self) -> None: # from there; the gap in between belongs to no session, which is the truth. if self.ledger is not None: self.ledger.release() + # Shape of the session for telemetry (FEAT-023): mode, cadence and tick + # count. Never the playbook, the journal, the pairs or the positions. + telemetry_taps.strategy_run( + self.config, + ticks=getattr(self.journal, "tick_count", 0) or 0, + stopped_by=self._last_stop_reason, + ) if self.journal: self.journal.close() _supervisor().unregister(self.agent_id, LoopState.STOPPED) @@ -282,6 +291,7 @@ async def _run_shutdown(self, reason: str) -> None: if self._shutting_down: return self._shutting_down = True + self._last_stop_reason = "shutdown" # Halt the loop so no next/concurrent tick fights the winddown. self._running = False self._paused = True @@ -338,19 +348,10 @@ def resume(self) -> None: self._paused = False _supervisor().record(self, LoopState.RUNNING) - def inject_directive(self, text: str) -> None: - """Queue a user directive to be included in the next tick's prompt.""" - self._pending_directives.append(text) - log.info("TickEngine %s: directive queued: %s", self.agent_id, text[:80]) - @property def is_running(self) -> bool: return self._running and self._task is not None and not self._task.done() - @property - def is_paused(self) -> bool: - return self._paused - @property def status(self) -> str: if not self._running: @@ -380,6 +381,16 @@ async def _loop(self) -> None: self.journal.append_error(str(e)) await self._notify(f"Agent {self.agent_id} tick error: {e}") + # A shutdown that started *inside* the tick (the hard risk + # kill-switch) already ran its winddown, wrote the terminal + # state and unregistered this run — it only returns here + # because it must not cancel its own task. Recording a tick + # now would rewrite state=running over that STOPPED, and the + # next boot would read a live run to reconcile and possibly + # restart the very strategy the risk engine just wound down. + if self._shutting_down or not self._running: + return + # Record the tick we just finished. A tiny atomic write, so if # the process dies mid-sleep the boot pass can say which tick # this run reached instead of guessing. @@ -394,6 +405,7 @@ async def _loop(self) -> None: label, ) await self._notify(f"Agent {self.agent_id}: {label} complete.") + self._last_stop_reason = "complete" self._running = False _supervisor().unregister(self.agent_id, LoopState.COMPLETED) return @@ -409,6 +421,7 @@ async def _loop(self) -> None: await self._notify( f"Agent {self.agent_id}: completed {max_ticks} ticks (max_ticks limit)." ) + self._last_stop_reason = "max_ticks" self._running = False self.journal.close() _supervisor().unregister(self.agent_id, LoopState.COMPLETED) @@ -490,12 +503,13 @@ async def _tick(self) -> None: return if risk_state.is_blocked and not self.is_experiment: - self.journal.append_action( - self.journal.tick_count + 1, - "tick_blocked", - risk_state.block_reason, - ) - self.journal.record_tick("blocked: " + risk_state.block_reason) + with self.journal.batch(): + self.journal.append_action( + self.journal.tick_count + 1, + "tick_blocked", + risk_state.block_reason, + ) + self.journal.record_tick("blocked: " + risk_state.block_reason) await self._notify( f"Agent {self.agent_id} blocked: {risk_state.block_reason}" ) @@ -571,12 +585,6 @@ async def _tick(self) -> None: canvas_nudge=canvas_nudge, ) - # Inject pending user directives - if self._pending_directives: - directives = "\n".join(f"- {d}" for d in self._pending_directives) - prompt += f"\n\nUSER DIRECTIVES (apply these on this tick):\n{directives}" - self._pending_directives.clear() - # 6. Create a fresh agent client per tick (clean context window) acp_client = await self._create_client(risk_state) self._active_client = acp_client @@ -586,8 +594,15 @@ async def _tick(self) -> None: tool_call_map: dict[str, dict[str, Any]] = {} await acp_client.start() + # Wall-clock budget for this tick's agent session. Comes from the shared + # policy (10 min default, CONDOR_TIMEOUT_TICK_DEFAULT) unless the run + # config sets ``tick_timeout_sec`` -- a slower model or a tick that does + # real research needs more room than a quoting loop does. + tick_timeout = resolve_tick_timeout( + execution_mode=mode, strategy=self.config.get("tick_timeout_sec") + ) try: - async with asyncio.timeout(300): + async with asyncio.timeout(tick_timeout): async for event in self._collect_stream(acp_client, prompt): if isinstance(event, TextChunk): response_chunks.append(event.text) @@ -596,7 +611,11 @@ async def _tick(self) -> None: if new_tc is not None: tool_calls.append(new_tc) except asyncio.TimeoutError: - log.warning("TickEngine %s: ACP prompt timed out", self.agent_id) + log.warning( + "TickEngine %s: ACP prompt timed out after %ds", + self.agent_id, + tick_timeout, + ) response_chunks.append("(timed out)") finally: await acp_client.stop() @@ -635,24 +654,40 @@ async def _tick(self) -> None: len(response_text), ) else: - # Sessions: full journal tracking - tick_num = self.journal.record_tick( - response_summary=response_text[:500], - ) + # Sessions: full journal tracking. Every journal.md update of this + # tick goes into one batch, so the file is rewritten once instead of + # three-to-five times (PERF-136). + with self.journal.batch(): + tick_num = self.journal.record_tick( + response_summary=response_text[:500], + ) - self._journal_ownership_violations(tick_num) - self._journal_mode_mismatch(tick_num) - - skill_pnl = self._last_skill_data.get("total_pnl", 0.0) - skill_volume = self._last_skill_data.get("total_volume", 0.0) - skill_executors = len(self._last_skill_data.get("executors", [])) - skill_exposure = self._last_skill_data.get("total_exposure", 0.0) - self.journal.record_snapshot( - total_pnl=skill_pnl, - total_volume=skill_volume, - open_count=skill_executors, - position_size=skill_exposure, - ) + self._journal_ownership_violations(tick_num) + self._journal_mode_mismatch(tick_num) + + skill_pnl = self._last_skill_data.get("total_pnl", 0.0) + skill_volume = self._last_skill_data.get("total_volume", 0.0) + skill_executors = len(self._last_skill_data.get("executors", [])) + skill_exposure = self._last_skill_data.get("total_exposure", 0.0) + self.journal.record_snapshot( + total_pnl=skill_pnl, + total_volume=skill_volume, + open_count=skill_executors, + position_size=skill_exposure, + ) + + action_brief = ( + response_text[:100].replace("\n", " ") + if response_text + else "No response" + ) + self.journal.write_summary( + tick=tick_num, + status="Running", + pnl=skill_pnl, + open_count=skill_executors, + last_action=action_brief, + ) self.journal.save_full_snapshot( tick=tick_num, @@ -665,19 +700,6 @@ async def _tick(self) -> None: duration=tick_duration, ) - action_brief = ( - response_text[:100].replace("\n", " ") - if response_text - else "No response" - ) - self.journal.write_summary( - tick=tick_num, - status="Running", - pnl=skill_pnl, - open_count=skill_executors, - last_action=action_brief, - ) - # Live session report (FEAT-036). Deterministic render over data we # already hold — no tokens. The guard is load-bearing: a charting or # report-index failure must never take down a trading tick. @@ -920,15 +942,42 @@ def _agent_key(self) -> str: ) def _resolve_server(self) -> tuple[str | None, dict | None]: - """Resolve the server for this agent.""" + """Resolve the server for this run, keyed on ``user_id`` (SEC-164). + + Every candidate — the configured name, the subject's own default, the + accessible list — is held to the same bar here: the server must exist + *and* ``user_id`` must be able to reach it. Two properties fall out of + checking it at resolution rather than only at start: + + * the run never inherits identity it was handed. ``chat_id`` arrives + from the request body, so resolving through ``get_effective_server`` + on *it* let a caller name a chat they do not own and pick up that + chat's default server, with no server name for the start gate + (SEC-149) to check. The subject's own default is keyed by user id + anyway, both on the web (``set_chat_default_server(user.id, ...)``) + and in a private Telegram chat, where chat id == user id. + * a name that matched nothing at start stays unusable. The loop is + long-running, so a configured ``server_name`` that resolved to + nothing — the ``"local"`` ``AgentConfig`` fills in, typically — must + not silently bind to whatever another user creates under that name + later. + + Falls back to the subject's accessible servers, as before, so a run + with no server configured still works. + """ from config_manager import get_config_manager, get_effective_server cm = get_config_manager() - server_name = self.config.get("server_name") - if not server_name: - server_name = get_effective_server(self.chat_id) - if not server_name: + def usable(name: str | None) -> bool: + return bool(name and cm.get_server(name)) and cm.has_server_access( + self.user_id, name + ) + + server_name = self.config.get("server_name") + if not usable(server_name): + server_name = get_effective_server(self.user_id) + if not usable(server_name): accessible = cm.get_accessible_servers(self.user_id) server_name = accessible[0] if accessible else None if not server_name: @@ -942,10 +991,18 @@ async def _get_client(self): try: server_name, server = self._resolve_server() if not server: - from handlers.bots._shared import get_bots_client - - client, _ = await get_bots_client(self.chat_id) - return client + # Nothing to fall back to: ``_resolve_server`` already tried + # every server this run's subject may use, so the only servers + # left are other people's. ``get_bots_client(self.chat_id)`` + # used to stand here — it ignores the chat id for server + # selection and, with no ``user_data`` to scope it, picks the + # first *globally* enabled server instead (SEC-164). + log.warning( + "Agent %s: no accessible server for user %s, no API client", + self.agent_id, + self.user_id, + ) + return None from config_manager import get_config_manager @@ -1002,6 +1059,10 @@ def get_info(self) -> dict[str, Any]: "close_type_counts": sd.get("close_type_counts", {}), "fees_known": sd.get("fees_known", True), "frequency_sec": self.config.get("frequency_sec", 60), + "tick_timeout_sec": resolve_tick_timeout( + execution_mode=self.config.get("execution_mode", "loop"), + strategy=self.config.get("tick_timeout_sec"), + ), "server_name": self.config.get("server_name", ""), "total_amount_quote": self.config.get("total_amount_quote", 100), "trading_context": self.config.get("trading_context", ""), diff --git a/condor/agents/journal.py b/condor/agents/journal.py index ce06e701d..d7a943add 100644 --- a/condor/agents/journal.py +++ b/condor/agents/journal.py @@ -22,10 +22,14 @@ import json import logging import re +from collections.abc import Iterator +from contextlib import contextmanager from datetime import datetime, timezone from pathlib import Path from typing import Any +from condor.fsutil import atomic_write_text + log = logging.getLogger(__name__) _DATA_ROOT = Path(__file__).parent.parent.parent / "agents" @@ -80,6 +84,30 @@ def resolve_agent_dirs(agent_id: str) -> tuple[Path | None, Path | None]: MAX_SNAPSHOTS = 100 +# Retention for the per-tick append-only sections of journal.md (PERF-136). +# Ticks and Snapshots gain one line per tick, so on a long-running session they +# grow without bound and every tick pays to rewrite them. Past the cap the +# oldest lines are *moved* -- not deleted -- to ``journal_archive.md`` next to +# journal.md, and the section keeps a marker line saying what went where. +# 1000 lines is ~17h of history at a 60s tick and ~4 days at a 5m tick. No +# reader of these two sections needs more: nothing reads a Ticks line's content +# at all (only the tick number, from the last one), and Snapshots feeds the +# drawdown peak -- carried across the trim in the marker -- plus a fallback +# equity curve. Everything else lives in Summary and Decisions, both already +# bounded and untouched here. +MAX_TICK_LINES = 1000 +MAX_SNAPSHOT_LINES = 1000 + +# Sidecar holding everything trimmed out of journal.md. Append-only, never read +# in the trading hot path: it exists so the audit trail survives the trim and a +# human (or the agent, via the journal_read MCP tool) can still go back. +ARCHIVE_NAME = "journal_archive.md" + +# Marker left in place of the archived lines. Deliberately starts with "- " so +# it reads as part of the list, and with a word that no entry parser accepts, so +# _parse_snapshots / _count_ticks skip it instead of parsing it as data. +ARCHIVE_MARKER_PREFIX = "- archived " + JOURNAL_TEMPLATE = """\ # Journal - {agent_id} @@ -189,12 +217,36 @@ def next_experiment_number(agent_dir: Path) -> int: TICK_LINE_PREFIX = "- tick#" +def highest_tick_number(text: str) -> int: + """Highest tick number recorded in ``text`` (0 if it holds no tick line). + + The tick *number* is read back from the line rather than derived from the + line count, so numbering survives the retention trim (PERF-136): a journal + whose first 1000 ticks were archived still resumes at 1001. Lines are + appended in order, so the last one normally wins, but the max is taken + anyway -- the MCP journal_write tool can edit the file out of band. + """ + highest = 0 + seen = 0 + for line in text.splitlines(): + if not line.startswith(TICK_LINE_PREFIX): + continue + seen += 1 + head = line[len(TICK_LINE_PREFIX) :].split("|", 1)[0].strip() + try: + highest = max(highest, int(head)) + except ValueError: + continue + # A tick line with an unparseable number is not a reason to restart from + # zero and overwrite history: fall back to counting what is there. + return highest or seen + + def count_journal_ticks(journal_path: Path) -> int: - """Count tick entries in a session's journal.md (lines record_tick writes).""" + """Number of ticks a session has run, from its journal.md.""" if not journal_path.exists(): return 0 - text = journal_path.read_text(errors="replace") - return sum(1 for line in text.splitlines() if line.startswith(TICK_LINE_PREFIX)) + return highest_tick_number(journal_path.read_text(errors="replace")) EXPERIMENT_TEMPLATE = """\ @@ -309,8 +361,6 @@ class JournalManager: def __init__( self, agent_id: str, - strategy_name: str = "", - strategy_description: str = "", session_dir: Path | None = None, agent_dir: Path | None = None, ): @@ -341,6 +391,14 @@ def __init__( self._cache_text: str = "" self._parsed_cache: dict[str, list[dict]] = {} + # Write batching (PERF-136): inside ``with journal.batch():`` the + # updated document is held here and hits the disk once, on exit. + self._batch_depth = 0 + self._pending_text: str | None = None + # Last section this process appended to journal_archive.md, so the + # sidecar gets one header per run of same-section lines, not per line. + self._archive_last_section: str | None = None + if not self._path.exists(): self._path.write_text(JOURNAL_TEMPLATE.format(agent_id=agent_id)) @@ -489,6 +547,9 @@ def _append_learning_to_journal(self, text_content: str) -> None: def read_full(self) -> str: """Return the entire journal contents (cached until the file changes).""" + if self._pending_text is not None: + # Mid-batch: the authoritative document is the one in memory. + return self._pending_text try: stat = self._path.stat() except OSError: @@ -500,9 +561,38 @@ def read_full(self) -> str: self._parsed_cache.clear() return self._cache_text + @contextmanager + def batch(self) -> Iterator[JournalManager]: + """Coalesce every journal write in the block into a single file write. + + A tick writes the journal three to five times (record_tick, + record_snapshot, write_summary, plus append_action on violations) and + each one used to rewrite the whole file. Inside this block the updates + are applied to an in-memory document -- readers inside the block see + them -- and the result is written once on exit, including on error, so + a tick that raises still journals what it had recorded. + """ + self._batch_depth += 1 + try: + yield self + finally: + self._batch_depth -= 1 + if self._batch_depth == 0: + self.flush() + + def flush(self) -> None: + """Write out a batched document, if any is pending.""" + text, self._pending_text = self._pending_text, None + if text is not None: + self._write_journal(text) + def _write_journal(self, text: str) -> None: """Write journal.md and invalidate the read cache.""" - self._path.write_text(text) + if self._batch_depth > 0: + self._pending_text = text + self._parsed_cache.clear() + return + atomic_write_text(self._path, text) self._cache_stamp = None self._parsed_cache.clear() @@ -798,8 +888,7 @@ def tick_count(self) -> int: return self._tick_count def _count_ticks(self) -> int: - section = self._get_section("Ticks") - return len([l for l in section.splitlines() if l.startswith(TICK_LINE_PREFIX)]) + return highest_tick_number(self._get_section("Ticks")) def record_tick(self, response_summary: str = "", actions: int = 0) -> int: """Record a tick entry. Returns the new tick number.""" @@ -810,7 +899,7 @@ def record_tick(self, response_summary: str = "", actions: int = 0) -> int: f"{TICK_LINE_PREFIX}{self._tick_count} | {now} " f"| actions={actions} | {summary}" ) - self._append_to_section("Ticks", entry) + self._append_bounded("Ticks", entry, MAX_TICK_LINES) return self._tick_count # ------------------------------------------------------------------ @@ -865,7 +954,7 @@ def record_snapshot( f"- {now} | pnl=${total_pnl:+.2f} | volume=${total_volume:,.0f} " f"| open={open_count} | exposure=${position_size:.2f}" ) - self._append_to_section("Snapshots", entry) + self._append_bounded("Snapshots", entry, MAX_SNAPSHOT_LINES) # ------------------------------------------------------------------ # Queries (used by RiskEngine) @@ -890,32 +979,6 @@ def _parse_executors(self) -> list[dict]: self._parsed_cache["executors"] = results return results - def _parse_ticks(self) -> list[dict]: - self.read_full() # refresh parsed cache if the file changed - cached = self._parsed_cache.get("ticks") - if cached is not None: - return cached - section = self._get_section("Ticks") - results = [] - for line in section.splitlines(): - if not line.startswith(TICK_LINE_PREFIX): - continue - entry: dict[str, Any] = {} - parts = line[2:].split(" | ") - for part in parts: - if part.startswith("tick#"): - entry["tick"] = int(part.replace("tick#", "")) - elif part.startswith("actions="): - entry["actions"] = int(part.replace("actions=", "")) - else: - if re.match(r"\d{4}-\d{2}-\d{2}", part.strip()): - entry["timestamp"] = part.strip() - else: - entry["summary"] = part.strip() - results.append(entry) - self._parsed_cache["ticks"] = results - return results - def _parse_snapshots(self) -> list[dict]: self.read_full() # refresh parsed cache if the file changed cached = self._parsed_cache.get("snapshots") @@ -924,24 +987,9 @@ def _parse_snapshots(self) -> list[dict]: section = self._get_section("Snapshots") results = [] for line in section.splitlines(): - if not line.startswith("- "): + if not line.startswith("- ") or line.startswith(ARCHIVE_MARKER_PREFIX): continue - entry: dict[str, Any] = {} - for part in line[2:].split(" | "): - part = part.strip() - if part.startswith("pnl=$"): - entry["pnl"] = float(part.replace("pnl=$", "").replace("+", "")) - elif part.startswith("volume=$"): - entry["volume"] = float( - part.replace("volume=$", "").replace(",", "") - ) - elif part.startswith("exposure=$"): - entry["exposure"] = float(part.replace("exposure=$", "")) - elif part.startswith("open="): - entry["open"] = int(part.replace("open=", "")) - elif re.match(r"\d{4}-\d{2}-\d{2}", part): - entry["timestamp"] = part - results.append(entry) + results.append(_parse_snapshot_line(line)) self._parsed_cache["snapshots"] = results return results @@ -976,8 +1024,11 @@ def get_drawdown_pct(self) -> float: if not snapshots: return 0.0 pnls = [s.get("pnl", 0) for s in snapshots] - peak = max(pnls) current = pnls[-1] + archived_peak = self._archived_peak_pnl() + if archived_peak is not None: + pnls.append(archived_peak) # the high-water mark outlives the trim + peak = max(pnls) drawdown = peak - current if drawdown <= 0: return 0.0 @@ -986,7 +1037,13 @@ def get_drawdown_pct(self) -> float: return 0.0 return drawdown / exposure * 100 - def get_pnl_series(self, hours: int = 24) -> list[dict]: + def get_pnl_series(self) -> list[dict]: + """PnL points still held in journal.md (at most MAX_SNAPSHOT_LINES). + + Older points are in ``journal_archive.md``; this is the equity-curve + fallback used when the bot's own series is empty, so a bounded tail is + what it needs -- it must not re-read the whole history every tick. + """ return [ {"timestamp": s.get("timestamp", ""), "pnl": s.get("pnl", 0)} for s in self._parse_snapshots() @@ -1036,6 +1093,72 @@ def _replace_section(self, name: str, content: str) -> None: new_text = text.rstrip() + f"\n\n## {name}\n{content}\n" self._write_journal(new_text) + def _append_bounded(self, section: str, entry: str, max_lines: int) -> None: + """Append a line to a section, keeping at most ``max_lines`` of them. + + Overflow is *moved* to ``journal_archive.md`` and replaced by a single + marker line naming the file and how much went into it, so the journal + never claims to be a complete record when it isn't. Costs the same one + read + one write as a plain append. + """ + lines = [l for l in self._get_section(section).splitlines() if l.strip()] + marker = "" + if lines and lines[0].startswith(ARCHIVE_MARKER_PREFIX): + marker = lines.pop(0) + lines.append(entry) + if len(lines) > max_lines: + cut = len(lines) - max_lines + dropped, keep = lines[:cut], lines[cut:] + try: + self._archive_lines(section, dropped) + except OSError: + # Could not park them safely -> do not drop them. The section + # stays one line over the cap and the trim retries next tick. + log.exception("journal: archiving %s failed, keeping lines", section) + else: + lines = keep + marker = _next_marker(section, marker, dropped) + body = [marker, *lines] if marker else lines + self._replace_section(section, "\n".join(body)) + + def _archive_lines(self, section: str, lines: list[str]) -> None: + """Append trimmed lines to the session's archive sidecar. + + Append-only and outside the journal's read path, so it costs O(lines + written) rather than O(history) -- the point of the trim. A failure + here must never take down a trading tick, but it must not silently + destroy the lines either, so it is logged loudly and the trim is + abandoned (the caller keeps them in journal.md for another round). + """ + path = self._session_dir / ARCHIVE_NAME + header = "" + if not path.exists(): + header = ( + f"# Journal archive - {self.agent_id}\n\n" + "Entries trimmed out of journal.md to keep it bounded. " + "Append-only; nothing here is ever rewritten.\n" + ) + if section != self._archive_last_section: + now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M") + header += f"\n## {section} (archived {now} UTC)\n" + with path.open("a", encoding="utf-8") as fh: + fh.write(header + "\n".join(lines) + "\n") + self._archive_last_section = section + + def _archived_peak_pnl(self) -> float | None: + """Peak PnL of the snapshots already archived, from the marker line. + + get_drawdown_pct measures the fall from the session's high-water mark, + so the trim must not lower that mark -- a forgotten peak would quietly + make the risk gate less protective than it was yesterday. + """ + for line in self._get_section("Snapshots").splitlines(): + if not line.startswith(ARCHIVE_MARKER_PREFIX): + break + m = re.search(r"peak_pnl=\$([+-]?[\d.]+)", line) + return float(m.group(1)) if m else None + return None + def _append_to_section(self, section: str, entry: str) -> None: """Append a line to a section.""" text = self.read_full() @@ -1064,13 +1187,58 @@ def entry_count(self) -> int: section = self._get_section("Recent Actions") return len([l for l in section.splitlines() if l.startswith("- ")]) - def get_data_dir(self) -> Path: - """Return the session data directory.""" - return self._session_dir - def size_bytes(self) -> int: - """Current file size.""" - return self._path.stat().st_size if self._path.exists() else 0 +def _parse_snapshot_line(line: str) -> dict[str, Any]: + """Parse one ``record_snapshot`` line into its fields.""" + entry: dict[str, Any] = {} + for part in line[2:].split(" | "): + part = part.strip() + if part.startswith("pnl=$"): + entry["pnl"] = float(part.replace("pnl=$", "").replace("+", "")) + elif part.startswith("volume=$"): + entry["volume"] = float(part.replace("volume=$", "").replace(",", "")) + elif part.startswith("exposure=$"): + entry["exposure"] = float(part.replace("exposure=$", "")) + elif part.startswith("open="): + entry["open"] = int(part.replace("open=", "")) + elif re.match(r"\d{4}-\d{2}-\d{2}", part): + entry["timestamp"] = part + return entry + + +def _next_marker(section: str, prev_marker: str, dropped: list[str]) -> str: + """Build the marker line that stands in for the archived entries. + + It is what a reader sees at the boundary, so it says how many entries are + missing and where they went -- and, for Snapshots, carries forward the peak + PnL that get_drawdown_pct would otherwise lose with the lines. + """ + prev_count = 0 + if prev_marker: + m = re.search(r"archived (\d+)", prev_marker) + prev_count = int(m.group(1)) if m else 0 + total = prev_count + len(dropped) + + if section == "Snapshots": + peaks = [ + entry["pnl"] + for entry in (_parse_snapshot_line(l) for l in dropped) + if "pnl" in entry + ] + m = re.search(r"peak_pnl=\$([+-]?[\d.]+)", prev_marker or "") + if m: + peaks.append(float(m.group(1))) + peak = f" | peak_pnl=${max(peaks):+.2f}" if peaks else "" + return ( + f"{ARCHIVE_MARKER_PREFIX}{total} earlier snapshot" + f"{'' if total == 1 else 's'} to {ARCHIVE_NAME}{peak}" + ) + + noun = section.rstrip("s").lower() + return ( + f"{ARCHIVE_MARKER_PREFIX}{total} earlier {noun}" + f"{'' if total == 1 else 's'} to {ARCHIVE_NAME}" + ) def _normalize(text: str) -> str: diff --git a/condor/agents/shutdown.py b/condor/agents/shutdown.py index 05432c374..a6ed6095a 100644 --- a/condor/agents/shutdown.py +++ b/condor/agents/shutdown.py @@ -20,6 +20,8 @@ import logging from typing import Any +from condor.runtime.timeouts import resolve_tick_timeout + from .strategy import Strategy, _parse_frontmatter log = logging.getLogger(__name__) @@ -298,10 +300,11 @@ async def _run_llm_cleanup( ) -> None: """Best-effort LLM nuance pass on top of the guaranteed deterministic floor. - Bounded by a hard 300s timeout (the same ceiling the tick ACP session runs - under) and fully fail-open: the safety-critical winddown already happened, so - any hang or error here is logged and swallowed — it can never strand a position - the way an LLM-only shutdown could. + Bounded by the same budget the tick agent session runs under (the shared + timeout policy, overridable per run with ``tick_timeout_sec``) and fully + fail-open: the safety-critical winddown already happened, so any hang or + error here is logged and swallowed — it can never strand a position the way + an LLM-only shutdown could. """ agent = getattr(engine, "agent", None) if not body or agent is None: @@ -312,7 +315,10 @@ async def _run_llm_cleanup( running = await _get_running_executors(engine, client) positions = await _fetch_positions(client, engine.agent_id) context = _build_llm_context(policy, running, positions, failures) - async with asyncio.timeout(300): + cleanup_timeout = resolve_tick_timeout( + strategy=engine.config.get("tick_timeout_sec") + ) + async with asyncio.timeout(cleanup_timeout): await _run_agent_to_completion( slug=agent.slug, user_id=engine.user_id, @@ -374,10 +380,11 @@ async def run_shutdown(engine: Any, reason: str) -> None: log.error(msg) await engine._notify(msg) if engine.journal: - engine.journal.append_action( - engine.journal.tick_count + 1, "shutdown_failed", "no API client" - ) - engine.journal.record_tick("shutdown failed (no client): " + reason) + with engine.journal.batch(): + engine.journal.append_action( + engine.journal.tick_count + 1, "shutdown_failed", "no API client" + ) + engine.journal.record_tick("shutdown failed (no client): " + reason) return stopped, failures = await _deterministic_baseline(engine, client, policy) @@ -406,9 +413,12 @@ async def run_shutdown(engine: Any, reason: str) -> None: if engine.journal: verified = "flat" if not stranded else f"{len(stranded)} stranded" - engine.journal.append_action( - engine.journal.tick_count + 1, - "shutdown_done", - f"stopped={stopped}, failures={len(failures)}, verify={verified}", - ) - engine.journal.record_tick("shutdown: " + reason) + # Both journal.md updates of this winddown go into one batch, so the file + # is rewritten once instead of twice (PERF-136 idiom, PERF-173). + with engine.journal.batch(): + engine.journal.append_action( + engine.journal.tick_count + 1, + "shutdown_done", + f"stopped={stopped}, failures={len(failures)}, verify={verified}", + ) + engine.journal.record_tick("shutdown: " + reason) diff --git a/condor/archived_pnl.py b/condor/archived_pnl.py index 4bc968679..f679a5f33 100644 --- a/condor/archived_pnl.py +++ b/condor/archived_pnl.py @@ -13,7 +13,7 @@ import logging from collections import defaultdict from datetime import datetime -from typing import Any +from typing import Any, NamedTuple logger = logging.getLogger(__name__) @@ -89,74 +89,78 @@ def calculate_pnl_from_trades(trades: list[dict[str, Any]]) -> dict[str, Any]: return _calculate_pnl_open_close(trades) -def _calculate_pnl_average_cost(trades: list[dict[str, Any]]) -> dict[str, Any]: - """ - Calculate PnL using average cost basis for spot/market making trades. +class _ParsedTrade(NamedTuple): + """One trade row after the field extraction every mode shares.""" + + pair: str + amount: float + price: float + trade_type: str + position: str + fee: float + timestamp: Any + + +class _TradeOutcome(NamedTuple): + """What an accounting mode decided about one trade. - This handles NIL position trades where: - - BUY adds to inventory at that price - - SELL realizes PnL based on weighted average cost of inventory + ``running_delta`` is added to the running total unconditionally. + ``pair_delta`` is credited to ``pnl_by_pair`` only when it is not ``None``, + so a mode that touches the running total *without* attributing the amount + to a pair (average cost does exactly that for the fee on an unmatched SELL) + does not create a spurious ``pnl_by_pair`` entry. """ - # Track inventory per trading pair using average cost - # inventory = {amount: float, total_cost: float} - inventory: dict[str, dict[str, float]] = {} - pnl_by_pair: dict[str, float] = defaultdict(float) - cumulative_pnl: list[dict[str, Any]] = [] - running_pnl = 0.0 - total_fees = 0.0 - total_volume = 0.0 + running_delta: float = 0.0 + pair_delta: float | None = None - # Debug counters - buy_count = 0 - sell_count = 0 - realized_trades = 0 - # Sort trades by timestamp - sorted_trades = sorted(trades, key=lambda t: t.get("timestamp", 0)) +_NO_CHANGE = _TradeOutcome() - for trade in sorted_trades: - pair = trade.get("trading_pair", "Unknown") - amount = float(trade.get("amount", 0)) - price = float(trade.get("price", 0)) - trade_type = trade.get("trade_type", "").upper() - fee = float(trade.get("trade_fee_in_quote", 0)) - timestamp = trade.get("timestamp", 0) - total_fees += fee - total_volume += amount * price +class _AverageCostAccounting: + """Average cost basis inventory tracking, for spot / market making (NIL). - # Parse timestamp for cumulative chart - ts = parse_timestamp(timestamp) + BUY adds to inventory at its price; SELL realizes against the weighted + average cost of the inventory on hand, clamped to that inventory. A SELL + with no inventory (a short) is not modelled: only its fee is charged, and + only to the running total. + """ + + def __init__(self) -> None: + # inventory = {amount: float, total_cost: float} + self.inventory: dict[str, dict[str, float]] = {} + self.buy_count = 0 + self.sell_count = 0 + self.realized_trades = 0 + def __call__(self, trade: _ParsedTrade) -> _TradeOutcome: # Initialize inventory for this pair if needed - if pair not in inventory: - inventory[pair] = {"amount": 0.0, "total_cost": 0.0} + if trade.pair not in self.inventory: + self.inventory[trade.pair] = {"amount": 0.0, "total_cost": 0.0} - inv = inventory[pair] + inv = self.inventory[trade.pair] - if trade_type == "BUY": - buy_count += 1 + if trade.trade_type == "BUY": + self.buy_count += 1 # Add to inventory at this price - inv["amount"] += amount - inv["total_cost"] += amount * price + inv["amount"] += trade.amount + inv["total_cost"] += trade.amount * trade.price + return _NO_CHANGE - elif trade_type == "SELL": - sell_count += 1 + if trade.trade_type == "SELL": + self.sell_count += 1 # Realize PnL if we have inventory if inv["amount"] > 0: - realized_trades += 1 + self.realized_trades += 1 # Calculate average cost of inventory avg_cost = inv["total_cost"] / inv["amount"] if inv["amount"] > 0 else 0 # Determine how much we can actually sell from inventory - sell_amount = min(amount, inv["amount"]) + sell_amount = min(trade.amount, inv["amount"]) # PnL = (sell_price - avg_cost) * amount - fee - pnl = (price - avg_cost) * sell_amount - fee - - pnl_by_pair[pair] += pnl - running_pnl += pnl + pnl = (trade.price - avg_cost) * sell_amount - trade.fee # Reduce inventory if sell_amount >= inv["amount"]: @@ -168,143 +172,183 @@ def _calculate_pnl_average_cost(trades: list[dict[str, Any]]) -> dict[str, Any]: ratio = sell_amount / inv["amount"] inv["amount"] -= sell_amount inv["total_cost"] -= inv["total_cost"] * ratio - else: - # Short selling (no inventory) - track as negative PnL for now - # This means we're selling something we don't have (going short) - # For simplicity, just count fees - running_pnl -= fee - # Record cumulative PnL point for charting - if ts: - cumulative_pnl.append( - { - "timestamp": ts, - "pnl": running_pnl, - "pair": pair, - } - ) + return _TradeOutcome(running_delta=pnl, pair_delta=pnl) - logger.info( - f"PnL calculation (avg cost): {len(trades)} trades, {buy_count} BUY, {sell_count} SELL, " - f"{realized_trades} realized, total_pnl=${running_pnl:.4f}" - ) + # Short selling (no inventory) - track as negative PnL for now + # This means we're selling something we don't have (going short) + # For simplicity, just count fees. Note the fee hits the running + # total but is deliberately NOT attributed to pnl_by_pair. + return _TradeOutcome(running_delta=-trade.fee) - return { - "total_pnl": running_pnl, - "total_fees": total_fees, - "pnl_by_pair": dict(pnl_by_pair), - "cumulative_pnl": cumulative_pnl, - "total_volume": total_volume, - } + return _NO_CHANGE + def summary(self, trade_count: int, running_pnl: float) -> str: + return ( + f"PnL calculation (avg cost): {trade_count} trades, {self.buy_count} BUY, " + f"{self.sell_count} SELL, {self.realized_trades} realized, " + f"total_pnl=${running_pnl:.4f}" + ) -def _calculate_pnl_open_close(trades: list[dict[str, Any]]) -> dict[str, Any]: - """ - Calculate PnL using OPEN/CLOSE position tracking for perpetual futures. - For perpetual futures: - - OPEN trades establish positions (long or short) - - CLOSE trades realize PnL - """ - # Track positions per trading pair - # position = {amount: float, total_cost: float, direction: int (1=long, -1=short)} - positions: dict[str, dict[str, Any]] = {} +class _OpenClosePositionAccounting: + """OPEN/CLOSE position tracking, for perpetual futures. - pnl_by_pair: dict[str, float] = defaultdict(float) - cumulative_pnl: list[dict[str, Any]] = [] - running_pnl = 0.0 - total_fees = 0.0 - total_volume = 0.0 - - # Debug counters - open_count = 0 - close_count = 0 - close_with_position = 0 - - # Sort trades by timestamp - sorted_trades = sorted(trades, key=lambda t: t.get("timestamp", 0)) + OPEN trades establish or add to a position (long on BUY, short on SELL); + CLOSE trades realize PnL against the average entry price. Unlike average + cost, a CLOSE realizes its full amount even when that exceeds the position, + and a CLOSE with no position is a complete no-op - not even its fee is + charged to the running total. + """ - for trade in sorted_trades: - pair = trade.get("trading_pair", "Unknown") - amount = float(trade.get("amount", 0)) - price = float(trade.get("price", 0)) - trade_type = trade.get("trade_type", "").upper() # BUY or SELL - position_action = trade.get("position", "").upper() # OPEN or CLOSE - fee = float(trade.get("trade_fee_in_quote", 0)) - timestamp = trade.get("timestamp", 0) + def __init__(self) -> None: + # position = {amount: float, total_cost: float, direction: int (1=long, -1=short)} + self.positions: dict[str, dict[str, Any]] = {} + self.open_count = 0 + self.close_count = 0 + self.close_with_position = 0 - total_fees += fee - total_volume += amount * price + def __call__(self, trade: _ParsedTrade) -> _TradeOutcome: + if trade.position == "OPEN": + self.open_count += 1 + # Opening a new position or adding to existing + if trade.pair not in self.positions: + self.positions[trade.pair] = { + "amount": 0, + "total_cost": 0, + "direction": 0, + } - # Parse timestamp for cumulative chart - ts = parse_timestamp(timestamp) + pos = self.positions[trade.pair] + pos["amount"] += trade.amount + pos["total_cost"] += trade.price * trade.amount + # 1 = long (BUY), -1 = short (SELL). Recorded for inspection only; + # the close side reads trade_type, not this field. + pos["direction"] = 1 if trade.trade_type == "BUY" else -1 + return _NO_CHANGE - if position_action == "OPEN": - open_count += 1 - # Opening a new position or adding to existing - if pair not in positions: - positions[pair] = {"amount": 0, "total_cost": 0, "direction": 0} - - pos = positions[pair] - - if trade_type == "BUY": - # Opening/adding to long position - pos["amount"] += amount - pos["total_cost"] += price * amount - pos["direction"] = 1 - else: # SELL - # Opening/adding to short position - pos["amount"] += amount - pos["total_cost"] += price * amount - pos["direction"] = -1 - - elif position_action == "CLOSE": - close_count += 1 + if trade.position == "CLOSE": + self.close_count += 1 # Closing a position - realize PnL - pos = positions.get(pair) + pos = self.positions.get(trade.pair) if pos and pos["amount"] > 0: - close_with_position += 1 + self.close_with_position += 1 # Calculate average entry price avg_entry = pos["total_cost"] / pos["amount"] - if trade_type == "SELL": + if trade.trade_type == "SELL": # Closing long: PnL = (exit - entry) * amount - pnl = (price - avg_entry) * amount + pnl = (trade.price - avg_entry) * trade.amount else: # BUY # Closing short: PnL = (entry - exit) * amount - pnl = (avg_entry - price) * amount + pnl = (avg_entry - trade.price) * trade.amount # Subtract fee from PnL - pnl -= fee - - pnl_by_pair[pair] += pnl - running_pnl += pnl + pnl -= trade.fee # Update position - if amount >= pos["amount"]: + if trade.amount >= pos["amount"]: # Fully closed - del positions[pair] + del self.positions[trade.pair] else: # Partially closed - close_ratio = amount / pos["amount"] - pos["amount"] -= amount + close_ratio = trade.amount / pos["amount"] + pos["amount"] -= trade.amount pos["total_cost"] -= pos["total_cost"] * close_ratio + return _TradeOutcome(running_delta=pnl, pair_delta=pnl) + + # CLOSE with nothing open: no PnL and, unlike average cost mode, + # no fee charged to the running total either. + return _NO_CHANGE + + return _NO_CHANGE + + def summary(self, trade_count: int, running_pnl: float) -> str: + return ( + f"PnL calculation (open/close): {trade_count} trades, {self.open_count} OPEN, " + f"{self.close_count} CLOSE, {self.close_with_position} CLOSE with matching " + f"position, total_pnl=${running_pnl:.4f}" + ) + + +def _timestamp_sort_key(trade: dict[str, Any]) -> tuple[int, float, str]: + """Total order over every timestamp shape the archive can hand us. + + Sorting on the raw field blew up with ``TypeError: '<' not supported + between instances of 'NoneType' and 'int'`` as soon as one row carried an + explicit ``timestamp: None`` next to a numeric one - a single malformed + archive row killed the whole PnL report instead of degrading. + + Grouping by shape first (numbers, then strings, then anything else) keeps + the ordering of a homogeneous list byte-identical to the naive sort, while + a mixed list becomes merely ordered rather than fatal. A null timestamp + sorts as ``0``, exactly like a missing one already did. + """ + ts = trade.get("timestamp") + if ts is None: + return (0, 0.0, "") + if isinstance(ts, (int, float)): # bool included, as before + return (0, float(ts), "") + if isinstance(ts, str): + return (1, 0.0, ts) + parsed = parse_timestamp(ts) # datetime / pandas Timestamp + if parsed is not None: + return (0, parsed.timestamp(), "") + return (2, 0.0, "") # unknown shape: stable, keeps input order + + +def _walk_trades(trades: list[dict[str, Any]], accounting) -> dict[str, Any]: + """Run the per-trade frame both PnL modes share. + + Sorts by timestamp, extracts the fields, accumulates fees and volume, + delegates the accounting decision to ``accounting``, and assembles the + cumulative series and the result dict. The accounting object owns all + mode-specific state and arithmetic; this function owns none of it. + """ + pnl_by_pair: dict[str, float] = defaultdict(float) + cumulative_pnl: list[dict[str, Any]] = [] + running_pnl = 0.0 + total_fees = 0.0 + total_volume = 0.0 + + # Sort trades by timestamp + sorted_trades = sorted(trades, key=_timestamp_sort_key) + + for trade in sorted_trades: + parsed = _ParsedTrade( + pair=trade.get("trading_pair", "Unknown"), + amount=float(trade.get("amount", 0)), + price=float(trade.get("price", 0)), + trade_type=trade.get("trade_type", "").upper(), # BUY or SELL + position=trade.get("position", "").upper(), # OPEN, CLOSE or NIL + fee=float(trade.get("trade_fee_in_quote", 0)), + timestamp=trade.get("timestamp"), + ) + + total_fees += parsed.fee + total_volume += parsed.amount * parsed.price + + # Parse timestamp for cumulative chart + ts = parse_timestamp(parsed.timestamp) + + outcome = accounting(parsed) + if outcome.pair_delta is not None: + pnl_by_pair[parsed.pair] += outcome.pair_delta + running_pnl += outcome.running_delta + # Record cumulative PnL point for charting if ts: cumulative_pnl.append( { "timestamp": ts, "pnl": running_pnl, - "pair": pair, + "pair": parsed.pair, } ) - logger.info( - f"PnL calculation (open/close): {len(trades)} trades, {open_count} OPEN, {close_count} CLOSE, " - f"{close_with_position} CLOSE with matching position, total_pnl=${running_pnl:.4f}" - ) + logger.info(accounting.summary(len(trades), running_pnl)) return { "total_pnl": running_pnl, @@ -313,3 +357,13 @@ def _calculate_pnl_open_close(trades: list[dict[str, Any]]) -> dict[str, Any]: "cumulative_pnl": cumulative_pnl, "total_volume": total_volume, } + + +def _calculate_pnl_average_cost(trades: list[dict[str, Any]]) -> dict[str, Any]: + """Calculate PnL using average cost basis for spot/market making trades.""" + return _walk_trades(trades, _AverageCostAccounting()) + + +def _calculate_pnl_open_close(trades: list[dict[str, Any]]) -> dict[str, Any]: + """Calculate PnL using OPEN/CLOSE position tracking for perpetual futures.""" + return _walk_trades(trades, _OpenClosePositionAccounting()) diff --git a/condor/asyncutil.py b/condor/asyncutil.py new file mode 100644 index 000000000..481e7bed6 --- /dev/null +++ b/condor/asyncutil.py @@ -0,0 +1,92 @@ +"""Shared asyncio helpers. + +``TaskSet`` is the one implementation of the fire-and-forget task tracker that +``condor`` had hand-rolled twice: once in ``condor/web/ws_manager.py`` +(CORR-107, one-shot broadcasts and backfills) and once in +``condor/server_data_service.py`` (CORR-141, subscriber callbacks). Both copies +existed because ``asyncio.create_task`` / ``ensure_future`` alone is unsafe for +work nobody awaits: + +* **The event loop only holds a weak reference**, so a task with no other + referent can be garbage-collected mid-await and silently disappear. +* **The exception is never retrieved**, so a crash surfaces only as asyncio's + "Task exception was never retrieved" at GC time — on no module's logger, with + nothing naming what stalled. + +``TaskSet`` holds the strong reference until the task completes, drops it +automatically, and reports failures on the *caller's* logger. Both call sites +word their error differently and identify the guilty task differently — the +ws manager by task name, SDS by ``subscriber_id`` — so the message template and +the label are the two things this helper parameterises rather than flattens. +Everything else (discard-then-check, cancellation is not a failure, snapshot +before cancelling because ``cancel()`` re-enters the done-callback) was +identical in both copies and is fixed here. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Iterator + + +class TaskSet: + """Strong references to fire-and-forget tasks, with their failures logged. + + ``error_message`` is a printf-style template taking exactly two ``%s``: the + label of the task and the exception. It is logged on ``logger``, so failures + land on the calling module's logger and not on this one. + """ + + def __init__(self, logger: logging.Logger, error_message: str) -> None: + self._logger = logger + self._error_message = error_message + self._tasks: set[asyncio.Task] = set() + + def track(self, task: asyncio.Task, label: str | None = None) -> asyncio.Task: + """Keep a strong reference to ``task`` until it finishes. + + ``label`` names the task in the failure log; it defaults to the task's + own name, which is what a caller that names its tasks wants. The + reference is dropped automatically on completion. Returns the task so a + caller can keep dispatching in one expression. + """ + self._tasks.add(task) + task.add_done_callback(lambda t, lbl=label: self._on_done(t, lbl)) + return task + + def cancel_all(self) -> None: + """Cancel every still-pending task and forget all of them. + + Iterates a snapshot: ``cancel()`` fires the done-callback, which mutates + the underlying set. + """ + for task in list(self._tasks): + if not task.done(): + task.cancel() + self._tasks.clear() + + def _on_done(self, task: asyncio.Task, label: str | None) -> None: + """Release the strong reference and surface the failure, if any.""" + self._tasks.discard(task) + if task.cancelled(): + return + exc = task.exception() + if exc is not None: + self._logger.error( + self._error_message, label or task.get_name(), exc, exc_info=exc + ) + + # -- Introspection (tests and log lines; not part of the hot path) -- + + def __len__(self) -> int: + return len(self._tasks) + + def __iter__(self) -> Iterator[asyncio.Task]: + return iter(self._tasks) + + def __contains__(self, task: object) -> bool: + return task in self._tasks + + def __bool__(self) -> bool: + return bool(self._tasks) diff --git a/condor/backtest_store.py b/condor/backtest_store.py index 77174a03c..f710f83b3 100644 --- a/condor/backtest_store.py +++ b/condor/backtest_store.py @@ -3,6 +3,7 @@ Each backtest is stored as an individual JSON file under data/backtests/. A lightweight index (_index.json) tracks task_id -> server mapping for fast listing. """ + from __future__ import annotations import json @@ -10,6 +11,8 @@ from pathlib import Path from typing import Any +from condor.fsutil import atomic_write_json + logger = logging.getLogger(__name__) _DATA_DIR = Path("data") / "backtests" @@ -23,7 +26,9 @@ class BacktestStore: def __init__(self, data_dir: Path = _DATA_DIR) -> None: self._dir = data_dir self._index_path = data_dir / "_index.json" - self._index: dict[str, dict[str, str]] = {} # task_id -> {server, ...light meta} + self._index: dict[str, dict[str, str]] = ( + {} + ) # task_id -> {server, ...light meta} self._dir.mkdir(parents=True, exist_ok=True) self._load_index() self._migrate_legacy() @@ -76,29 +81,24 @@ def _read_file(self, task_id: str) -> dict[str, Any] | None: if not path.exists(): return None try: - return json.loads(path.read_text()) + return json.loads(path.read_text(encoding="utf-8")) except Exception: logger.warning("Failed to read backtest file %s", path) return None def _write_file(self, task_id: str, data: dict[str, Any]) -> None: - path = self._task_path(task_id) - tmp = path.with_suffix(".tmp") - tmp.write_text(json.dumps(data)) - tmp.replace(path) + atomic_write_json(self._task_path(task_id), data) def _load_index(self) -> None: if self._index_path.exists(): try: - self._index = json.loads(self._index_path.read_text()) + self._index = json.loads(self._index_path.read_text(encoding="utf-8")) except Exception: logger.warning("Failed to load backtest index, rebuilding") self._rebuild_index() def _persist_index(self) -> None: - tmp = self._index_path.with_suffix(".tmp") - tmp.write_text(json.dumps(self._index)) - tmp.replace(self._index_path) + atomic_write_json(self._index_path, self._index) def _rebuild_index(self) -> None: """Rebuild index from individual files on disk.""" @@ -107,11 +107,15 @@ def _rebuild_index(self) -> None: if path.name == "_index.json": continue try: - data = json.loads(path.read_text()) + data = json.loads(path.read_text(encoding="utf-8")) task_id = path.stem self._index[task_id] = { "server": data.get("server", ""), - "config": data.get("config", {}).get("id", "") if isinstance(data.get("config"), dict) else "", + "config": ( + data.get("config", {}).get("id", "") + if isinstance(data.get("config"), dict) + else "" + ), } except Exception: logger.warning("Skipping corrupt backtest file %s", path) @@ -122,17 +126,23 @@ def _migrate_legacy(self) -> None: if not _LEGACY_FILE.exists(): return try: - legacy_data = json.loads(_LEGACY_FILE.read_text()) + legacy_data = json.loads(_LEGACY_FILE.read_text(encoding="utf-8")) if not isinstance(legacy_data, dict) or not legacy_data: _LEGACY_FILE.unlink() return - logger.info("Migrating %d backtest results from legacy store", len(legacy_data)) + logger.info( + "Migrating %d backtest results from legacy store", len(legacy_data) + ) for task_id, entry in legacy_data.items(): server = entry.pop("server", "") self._write_file(task_id, {"server": server, **entry}) self._index[task_id] = { "server": server, - "config": entry.get("config", {}).get("id", "") if isinstance(entry.get("config"), dict) else "", + "config": ( + entry.get("config", {}).get("id", "") + if isinstance(entry.get("config"), dict) + else "" + ), } self._persist_index() # Remove legacy file after successful migration diff --git a/condor/backtesting.py b/condor/backtesting.py index 5a099816a..28aa7e071 100644 --- a/condor/backtesting.py +++ b/condor/backtesting.py @@ -15,6 +15,8 @@ import time from typing import Any +from condor.fetchers.executors import normalize_executor_side + logger = logging.getLogger(__name__) # Polling defaults for run_and_save. The timeout is deliberately generous: a caller @@ -57,6 +59,32 @@ def coerce_controller_config(config: dict) -> dict: return out +def normalize_backtest_task(task: Any) -> Any: + """Normalize the executor sides inside a backtest envelope, in place. + + The backtesting engine reports a side the way the raw API does -- ``1``, + ``TradeType.BUY``, ``LONG`` -- and this payload is the one wire that never went + through :func:`condor.fetchers.executors.normalize_executor_side`, so the + dashboard carried a TypeScript reimplementation of the rule to render a backtest + (ARCH-121). Applying the canonical normalizer here makes the backtest wire match + the executor wire, leaving exactly one definition of the rule. + + Accepts either the task envelope (``{status, config, result, ...}``) or a bare + result payload, and only rewrites a ``side`` that is actually present, so it adds + nothing to the raw payload the dashboard also renders as JSON. + """ + if not isinstance(task, dict): + return task + result = task.get("result") + payload = result if isinstance(result, dict) else task + executors = payload.get("executors") + if isinstance(executors, list): + for ex in executors: + if isinstance(ex, dict) and "side" in ex: + ex["side"] = normalize_executor_side(ex["side"]) + return task + + async def run_and_save( client, server: str, @@ -130,7 +158,7 @@ async def _poll_task( f"Backtest {task_id} returned an unreadable task: {task}" ) if task.get("status") in _TERMINAL: - return task + return normalize_backtest_task(task) if time.monotonic() >= deadline: raise BacktestError( f"Backtest {task_id} is still {task.get('status') or 'running'} after " diff --git a/condor/cache.py b/condor/cache.py index ce1262e66..783a0cfac 100644 --- a/condor/cache.py +++ b/condor/cache.py @@ -125,9 +125,7 @@ def evict_expired( del cache[k] if stale_keys: - logger.debug( - "Evicted %d stale entries from '%s'", len(stale_keys), namespace - ) + logger.debug("Evicted %d stale entries from '%s'", len(stale_keys), namespace) return len(stale_keys) diff --git a/condor/code_runner.py b/condor/code_runner.py new file mode 100644 index 000000000..752843c85 --- /dev/null +++ b/condor/code_runner.py @@ -0,0 +1,244 @@ +"""Run an ad-hoc Python snippet with a routine's primitives (FEAT-047). + +This is the thinner sibling of ``RoutineStore._execute_and_record``: same main +process, same :class:`WebRoutineContext`, same report attribution — with the +routine file and its config schema removed. What a snippet can reach, a routine +can reach, and vice versa; that is the point, because "it worked in run_code" is +otherwise a bug class. + +The snippet contract is deliberately *no* contract: the code is a script body, +top-level ``await`` is allowed, ``print()`` is the output and an optional +``result`` variable is the return value. ``PyCF_ALLOW_TOP_LEVEL_AWAIT`` buys +that without wrapping the source in a function, which is what keeps traceback +line numbers pointing at the line the caller actually wrote — the single most +useful property when the caller is a model about to fix its own snippet. + +There is no sandbox and that is deliberate — a snippet is meant to reach exactly +what a routine reaches. It follows that calling ``execute_code`` is handing over +the whole process, so every caller must gate it at least as tightly as +``condor/web/routes/code.py`` does (SEC-151); "the caller is authenticated" is +not a gate. +""" + +from __future__ import annotations + +import ast +import asyncio +import contextvars +import inspect +import io +import logging +import sys +import time +import traceback + +import condor.reports as reports +from condor.code_runs import CodeRun, get_code_run_store, new_run_id +from condor.memory.paths import CHAT_SLUG + +logger = logging.getLogger(__name__) + +# The name compiled snippets carry, and therefore the name in their tracebacks. +CODE_FILENAME = "" + +DEFAULT_TIMEOUT = 60.0 +MAX_TIMEOUT = 120.0 + +# Where a run's ``print`` goes. Per-context, so concurrent runs cannot read each +# other's output — which only holds because every run gets its own asyncio.Task +# (a Task copies the context; awaiting a bare coroutine would share ours and +# leak the buffer into the caller). +_CAPTURE: contextvars.ContextVar[io.StringIO | None] = contextvars.ContextVar( + "code_run_stdout", default=None +) + + +class _StdoutProxy: + """``sys.stdout`` that routes to the running snippet's buffer, else onward. + + Installed process-wide and never removed, so it must be invisible to + everything that is not a code run: with no capture buffer set it is a pass + through to the stream it wrapped, and the bot's own output is untouched. + """ + + def __init__(self, stream): + self._stream = stream + + def write(self, data): + buffer = _CAPTURE.get() + if buffer is not None: + return buffer.write(data) + return self._stream.write(data) + + def flush(self): + buffer = _CAPTURE.get() + if buffer is None: + self._stream.flush() + + def __getattr__(self, name): + return getattr(self._stream, name) + + +def _install_stdout_proxy() -> None: + """Wrap ``sys.stdout`` once. The isinstance check is the idempotence. + + Not a "did we already do it" flag: something else may have replaced + ``sys.stdout`` since (a test harness, a debugger), and then the wrapper is + gone and capture has to be re-established over the new stream. + """ + if not isinstance(sys.stdout, _StdoutProxy): + sys.stdout = _StdoutProxy(sys.stdout) + + +def _trim_traceback(text: str) -> str: + """Drop the harness frames above the snippet's own. + + Everything before the first ``File ""`` line is this module + calling ``eval``; a model reading the traceback to fix its code should see + only its code. + """ + lines = text.splitlines(keepends=True) + for i, line in enumerate(lines): + if CODE_FILENAME in line: + head = lines[0] if lines and lines[0].startswith("Traceback") else "" + return head + "".join(lines[i:]) + return text + + +def _render_result(namespace: dict) -> str: + """The snippet's ``result`` global as text, or "" when it never set one. + + ``str`` rather than ``repr``: the readers are a model and a human, and + ``2.0`` says what ``np.float64(2.0)`` says with less noise, while a + DataFrame renders as its table either way. + """ + if "result" not in namespace: + return "" + value = namespace["result"] + if value is None: + return "" + try: + return str(value) + except Exception as e: # a __str__ that raises must not fail the run + return f"" + + +async def _run_in_task( + compiled, namespace: dict, buffer, agent: str, source: str, out: dict +): + """The snippet itself, in its own task context. + + Setting the capture ContextVar *here* (not in the caller) is what keeps the + buffer private to this run, and the report id is read in a ``finally`` so a + snippet that saved a report and then raised still reports it. + """ + _CAPTURE.set(buffer) + reports.reset_last_report_id() + try: + with reports.attribute_to(agent), reports.default_source("code", source): + outcome = eval(compiled, namespace) # noqa: S307 - this IS the feature + if inspect.isawaitable(outcome): + await outcome + finally: + out["report_id"] = reports.get_last_report_id() or "" + + +async def execute_code( + code: str, + *, + server_name: str = "", + agent: str = "", + label: str = "", + timeout: float = DEFAULT_TIMEOUT, + chat_id: int = 0, + session_key: str = "", +) -> dict: + """Compile, run and persist one snippet. Returns the stored record as a dict.""" + from condor.routine_store import WebRoutineContext, get_routine_store + from config_manager import get_client + + run_id = new_run_id() + agent = agent or CHAT_SLUG + timeout = max(1.0, min(float(timeout or DEFAULT_TIMEOUT), MAX_TIMEOUT)) + buffer = io.StringIO() + started = time.monotonic() + + run = CodeRun( + id=run_id, + created=time.time(), + agent=agent, + label=label, + server=server_name, + code=code, + session_key=session_key, + ) + + try: + compiled = compile( + code, CODE_FILENAME, "exec", flags=ast.PyCF_ALLOW_TOP_LEVEL_AWAIT + ) + except SyntaxError as e: + run.status = "error" + run.error = f"{type(e).__name__}: {e}" + run.traceback = _trim_traceback(traceback.format_exc()) + run.duration_ms = int((time.monotonic() - started) * 1000) + get_code_run_store().save(run) + return run.to_dict() + + # The exact context a routine gets, so the two paths cannot diverge — the + # active server travels as a preference, which is how get_client resolves it. + context = WebRoutineContext( + server_name, bot=get_routine_store().get_bot(), chat_id=chat_id + ) + # Best effort: a pure-pandas snippet must still run when no server resolves, + # and the reason belongs in the run's own output, not in a raised error. + try: + client = await get_client(chat_id, context=context) + except Exception as e: + client = None + buffer.write(f"[run_code] no API client available ({e}); `client` is None\n") + + namespace = { + "__name__": "__code_run__", + "context": context, + "client": client, + } + + _install_stdout_proxy() + out: dict = {"report_id": ""} + task = asyncio.create_task( + _run_in_task(compiled, namespace, buffer, agent, label or run_id, out) + ) + try: + await asyncio.wait_for(task, timeout=timeout) + run.status = "ok" + except (asyncio.TimeoutError, TimeoutError): + run.status = "timeout" + run.error = f"Timed out after {timeout:g}s" + except asyncio.CancelledError: + run.status = "timeout" + run.error = "Cancelled" + raise + except BaseException as e: + run.status = "error" + run.error = f"{type(e).__name__}: {e}" + run.traceback = _trim_traceback( + "".join(traceback.format_exception(type(e), e, e.__traceback__)) + ) + finally: + run.duration_ms = int((time.monotonic() - started) * 1000) + run.stdout = buffer.getvalue() + run.result = _render_result(namespace) + run.report_id = out.get("report_id", "") + # Never log the snippet or its output: a run can hold credentials it + # printed, and the application log has neither the cap nor the gitignore + # that data/code_runs/ has. + logger.info( + "code run %s finished status=%s in %dms", + run_id, + run.status, + run.duration_ms, + ) + get_code_run_store().save(run) + + return run.to_dict() diff --git a/condor/code_runs.py b/condor/code_runs.py new file mode 100644 index 000000000..d34e51e1d --- /dev/null +++ b/condor/code_runs.py @@ -0,0 +1,166 @@ +"""Persistence for ad-hoc code runs (FEAT-047). + +Same shape as :mod:`condor.backtest_store` and the report index: one JSON file +per run under ``data/code_runs/`` plus a light ``_index.json`` so listing a +history needs no per-file reads. No sqlite anywhere in this repo, and a store +that holds a few hundred short records whose main consumer reads one record at +a time does not earn the first one. + +The store keeps the **full** text of every run — code, stdout, result, +traceback. Clipping for a model's context is the caller's job, so a snippet +that failed can always be read back whole and corrected. +""" + +from __future__ import annotations + +import json +import logging +import re +import secrets +import time +from dataclasses import asdict, dataclass +from pathlib import Path + +from condor.fsutil import atomic_write_json + +logger = logging.getLogger(__name__) + +# Anchored at the repo root, not the cwd: the main process writes this dir and +# every MCP subprocess reads it, and they need not agree on a working directory. +_DATA_DIR = Path(__file__).resolve().parents[1] / "data" / "code_runs" + +# Retention bound. Also the only thing standing between a snippet that printed a +# credential and an unbounded pile of them on disk (see FEAT-047 risks). +MAX_RUNS = 300 + +# Ids are generated here, but they also arrive from a tool argument — anything +# that is not this shape never becomes a path. +_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$") + +# The light meta the index carries, so `list` never opens a run file. +_INDEX_FIELDS = ("id", "created", "agent", "label", "status", "error", "duration_ms") + + +@dataclass +class CodeRun: + """One execution of an ad-hoc snippet, exactly as it is persisted.""" + + id: str + created: float + agent: str = "" # attribution slug ("condor" | agent slug) + label: str = "" # short purpose the caller passed ("returns of SOL 1h") + server: str = "" + status: str = "ok" # "ok" | "error" | "timeout" + duration_ms: int = 0 + code: str = "" + stdout: str = "" + result: str = "" # the snippet's `result` global, rendered; "" if unset + error: str = "" # "ZeroDivisionError: division by zero" + traceback: str = "" + report_id: str = "" + session_key: str = "" + + def to_dict(self) -> dict: + return asdict(self) + + +def new_run_id() -> str: + """``code__<16 hex>`` — sortable by time, unique under concurrency. + + The suffix carries 64 bits: runs starting in the same millisecond are common + (a loop, a burst of tool calls) and the store is keyed by this id, so a + collision silently overwrites the earlier run instead of raising. + """ + return f"code_{int(time.time() * 1000)}_{secrets.token_hex(8)}" + + +class CodeRunStore: + """Per-file JSON persistence for code runs, newest-first index, capped.""" + + def __init__(self, data_dir: Path | None = None) -> None: + self._dir = Path(data_dir) if data_dir is not None else _DATA_DIR + self._index_path = self._dir / "_index.json" + self._index: list[dict] = [] + self._dir.mkdir(parents=True, exist_ok=True) + self._load_index() + + # ── public API ── + + def save(self, run: CodeRun) -> None: + """Write the run, put it at the head of the index, evict past the cap.""" + data = run.to_dict() + self._write_json(self._run_path(run.id), data) + self._index = [e for e in self._index if e.get("id") != run.id] + self._index.insert(0, {k: data.get(k) for k in _INDEX_FIELDS}) + self._evict() + self._persist_index() + + def get(self, run_id: str) -> dict | None: + """The full record, or None if it is unknown or unreadable.""" + path = self._run_path(run_id) + if path is None or not path.exists(): + return None + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + logger.warning("Failed to read code run file %s", path) + return None + + def list(self, agent: str | None = None, limit: int = 20) -> list[dict]: + """Index meta only, newest first, optionally for one agent.""" + entries = self._index + if agent: + entries = [e for e in entries if e.get("agent") == agent] + return [dict(e) for e in entries[: max(0, limit)]] + + # ── internals ── + + def _run_path(self, run_id: str) -> Path | None: + """The file for ``run_id``, or None when the id is not a legal id. + + Ids reach this store from a tool argument, so the shape check is what + keeps ``get`` from being turned into an arbitrary file reader. + """ + if not run_id or not _ID_RE.match(run_id): + return None + return self._dir / f"{run_id}.json" + + def _evict(self) -> None: + """Drop the oldest runs past the cap — index entry *and* file.""" + for entry in self._index[MAX_RUNS:]: + path = self._run_path(entry.get("id", "")) + if path is not None and path.exists(): + try: + path.unlink() + except OSError: + logger.warning("Could not evict code run file %s", path) + del self._index[MAX_RUNS:] + + def _load_index(self) -> None: + if not self._index_path.exists(): + return + try: + loaded = json.loads(self._index_path.read_text(encoding="utf-8")) + self._index = loaded if isinstance(loaded, list) else [] + except Exception: + logger.warning("Failed to load code run index, starting empty") + self._index = [] + + def _persist_index(self) -> None: + self._write_json(self._index_path, self._index) + + def _write_json(self, path: Path | None, data) -> None: + """Atomic create-or-replace, via the shared :mod:`condor.fsutil` helper.""" + if path is None: + return + atomic_write_json(path, data, ensure_ascii=False) + + +_store: CodeRunStore | None = None + + +def get_code_run_store() -> CodeRunStore: + global _store + if _store is None: + _store = CodeRunStore() + return _store diff --git a/condor/dex_candles.py b/condor/dex_candles.py index 3cff51c62..86bed8c57 100644 --- a/condor/dex_candles.py +++ b/condor/dex_candles.py @@ -39,7 +39,7 @@ def uses_gecko_candles(connector: str) -> bool: """Whether this connector's candles come from GeckoTerminal, not the API.""" - from handlers.dex.pool_data import NETWORK_TO_GECKO + from condor.pool_data import NETWORK_TO_GECKO return connector in NETWORK_TO_GECKO @@ -62,7 +62,7 @@ async def _pool_ohlcv( use_cache: bool = True, ) -> list: """Raw OHLCV rows for one pool, or [] on any miss. Never raises.""" - from handlers.dex.pool_data import fetch_ohlcv + from condor.pool_data import fetch_ohlcv try: rows, err = await fetch_ohlcv( @@ -98,7 +98,7 @@ async def _resolve_pool(connector: str, trading_pair: str) -> tuple[str, str]: ticker pair on a network whose bases are addresses, for instance, where a lookup would only spend a request to fail. """ - from handlers.dex.pool_data import ( + from condor.pool_data import ( fetch_pair_top_pool, fetch_token_top_pool, uses_symbol_pairs, @@ -141,11 +141,7 @@ async def fetch_dex_candles( Returns candle dicts (``timestamp``/``open``/``high``/``low``/``close``/ ``volume``), ascending. """ - from handlers.dex.pool_data import ( - candles_needed, - normalize_timeframe, - timeframe_seconds, - ) + from condor.pool_data import candles_needed, normalize_timeframe, timeframe_seconds timeframe = normalize_timeframe(interval) tf_seconds = timeframe_seconds(timeframe) diff --git a/condor/fetchers/__init__.py b/condor/fetchers/__init__.py index 36e1adccc..e305cf52c 100644 --- a/condor/fetchers/__init__.py +++ b/condor/fetchers/__init__.py @@ -34,46 +34,49 @@ façade consumer; they are not the package's public surface. """ -from condor.fetchers.portfolio import fetch_portfolio -from condor.fetchers.positions import fetch_positions -from condor.fetchers.orders import fetch_active_orders -from condor.fetchers.trading_rules import fetch_trading_rules +from condor.fetchers.bots import fetch_bot_runs, fetch_bots_status from condor.fetchers.connectors import ( - fetch_connectors, fetch_available_cex_connectors, + fetch_connectors, + fetch_venues, is_cex_connector, ) from condor.fetchers.executors import ( - fetch_executors, - fetch_all_executors, create_executor, - stop_executor, describe_executor_error, + extract_executors_list, + fetch_all_executors, + fetch_executors, get_executor_detail, - get_executor_type, + get_executor_fees, get_executor_pnl, + get_executor_type, get_executor_volume, - get_executor_fees, - extract_executors_list, + stop_executor, ) -from condor.fetchers.bots import fetch_bots_status, fetch_bot_runs from condor.fetchers.market_data import ( - fetch_current_price, - fetch_candles, fetch_candle_connectors, + fetch_candles, + fetch_current_price, fetch_rates, fetch_ticker_pool, fetch_tickers, ) +from condor.fetchers.orders import fetch_active_orders +from condor.fetchers.portfolio import fetch_portfolio, fetch_portfolio_history +from condor.fetchers.positions import fetch_positions from condor.fetchers.server_status import fetch_server_status +from condor.fetchers.trading_rules import fetch_trading_rules __all__ = [ "fetch_portfolio", + "fetch_portfolio_history", "fetch_positions", "fetch_active_orders", "fetch_trading_rules", "fetch_connectors", "fetch_available_cex_connectors", + "fetch_venues", "is_cex_connector", "fetch_executors", "fetch_all_executors", diff --git a/condor/fetchers/connectors.py b/condor/fetchers/connectors.py index 6d4cb144a..720dc0b6e 100644 --- a/condor/fetchers/connectors.py +++ b/condor/fetchers/connectors.py @@ -1,14 +1,21 @@ """Fetch connector information from Hummingbot API.""" import logging -from typing import List +from typing import Dict, List from condor.fetchers._identifiers import validate_identifier +from condor.pool_data import NETWORK_TO_GECKO, network_has_clmm logger = logging.getLogger(__name__) _DEX_PREFIXES = ( - "solana", "ethereum", "polygon", "arbitrum", "base", "optimism", "avalanche", + "solana", + "ethereum", + "polygon", + "arbitrum", + "base", + "optimism", + "avalanche", ) @@ -23,6 +30,35 @@ async def fetch_connectors(client, **_kw) -> List[str]: return await client.connectors.list_connectors() +def _network_id(item) -> str: + """The network id of a ``list_networks`` entry, whatever shape it arrives in. + + Gateway has returned plain strings and ``{"network_id": ...}`` / ``{"id": ...}`` + dicts across versions; the Telegram swap flow normalizes the same three shapes + (``handlers/dex/swap.py``). + """ + if isinstance(item, dict): + return str(item.get("network_id") or item.get("id") or item) + return str(item) + + +async def _chartable_gateway_networks(client) -> List[str]: + """Gateway networks Condor can chart: ``list_networks()`` ∩ ``NETWORK_TO_GECKO``. + + The intersection is the point: a network is only offered to the trade panel if + ``dex_candles.uses_gecko_candles`` will answer for it, so selecting one can + never produce an empty chart. Do not widen this to every gateway network. + + Raises whatever the gateway request raises — the caller decides how much of the + answer a gateway failure is allowed to take down. + """ + response = await client.gateway.list_networks() + networks = (response or {}).get("networks") or [] + return sorted( + {n for n in (_network_id(i) for i in networks) if n in NETWORK_TO_GECKO} + ) + + async def fetch_available_cex_connectors( client, account_name: str = "master_account", strict: bool = False, **_kw ) -> List[str]: @@ -61,3 +97,72 @@ async def fetch_available_cex_connectors( raise logger.error("Error fetching connectors: %s", e, exc_info=True) return [] + + +async def fetch_venues( + client, account_name: str = "master_account", strict: bool = False, **_kw +) -> List[Dict]: + """Venues the trade panel can offer, each with its independent traits. + + Returns ``[{"name", "hummingbot_market_data", "clmm_lp"}, ...]`` sorted by name. + The two traits are *independent facts about the venue*, each with exactly one + authority, because the panel's four decisions (which executor tabs, which + execution strategies, whether the order-book/rules/price endpoints are called, + where the current price comes from) are four different questions. Answering them + from a single ``cex``/``dex`` membership test is what this shape replaces. + + ``hummingbot_market_data``: the venue came from the credentialed connector list. + Credentials are per Hummingbot connector, so ``solana-mainnet-beta`` can never + appear there and presence is positive evidence that ``/market/trading-rules``, + ``/market/order-book``, ``/market/tickers`` and ``/market/prices`` answer. + **A venue in both input lists is a Hummingbot connector.** That is the ``xrpl`` + guarantee and this is the only place it is expressed: ``xrpl`` is a first-class + connector with a real order book whose *charts* come from GeckoTerminal, so it + sits in ``NETWORK_TO_GECKO`` and would appear in the gateway half the day a + Gateway version reports it — without this rule it would silently lose its order + book, its trading rules and every strategy but MARKET. + + ``clmm_lp``: the venue is a gateway network *and* its gecko chain hosts a CLMM + venue (``pool_data.network_has_clmm``), not merely that it is a gateway network. + That is what keeps an order-book venue from being offered an LP tab it cannot + honor. + + The candle source (CandlesFactory vs GeckoTerminal) is deliberately *not* a + field: the server already forks on it internally and no UI decision needs it. + + Args: + strict: Raise instead of reporting a venue-less server, so a failure is + never cached as "there is nothing here". Credentials are load-bearing, + so their failure always re-raises under ``strict``. A gateway failure + only re-raises when it would leave the list empty: losing the DEX half + must not empty the whole dropdown, and a list that still carries the + credentialed venues is not a cached lie. + """ + # Credentials first: under strict this raises, and there is no point asking the + # gateway about a server we cannot describe at all. + connectors = await fetch_available_cex_connectors( + client, account_name=account_name, strict=strict + ) + + gateway_error = None + networks: List[str] = [] + try: + networks = await _chartable_gateway_networks(client) + except Exception as e: + gateway_error = e + logger.error("Error fetching gateway networks: %s", e, exc_info=True) + + credentialed = set(connectors) + gateway = set(networks) + venues = [ + { + "name": name, + "hummingbot_market_data": name in credentialed, + "clmm_lp": name in gateway and network_has_clmm(name), + } + for name in sorted(credentialed | gateway) + ] + + if strict and gateway_error is not None and not venues: + raise gateway_error + return venues diff --git a/condor/fetchers/executors.py b/condor/fetchers/executors.py index 3142ea35a..2e37bc19d 100644 --- a/condor/fetchers/executors.py +++ b/condor/fetchers/executors.py @@ -42,12 +42,28 @@ def get_executor_type(executor: Dict[str, Any]) -> str: """Determine executor type from its data. Returns the executor type label (e.g. 'grid', 'position', 'order', 'dca', 'lp'). + + Never raises: this runs while building every display row, so an executor + whose ``config`` is explicitly ``null`` (a shape the backend does emit) must + degrade to ``unknown`` for that one row instead of taking down the whole + listing. """ - config = executor.get("config", executor) + # ``config`` is often present *and* null, so a ``dict.get`` default is not + # enough -- it only covers the absent key. Same isinstance guard as + # :func:`build_executor_row`, falling back to the executor itself so a row + # that carries its fields at the top level still resolves as before. + config = executor.get("config") + if not isinstance(config, dict): + config = executor for source in (config, executor): ex_type = source.get("type", "") or source.get("executor_type", "") if isinstance(ex_type, str) and ex_type: - label = ex_type.lower().replace("_executor", "").replace("executor", "").strip("_") + label = ( + ex_type.lower() + .replace("_executor", "") + .replace("executor", "") + .strip("_") + ) if label: return label if "start_price" in config and "end_price" in config: @@ -86,8 +102,13 @@ def normalize_executor_side(raw: Any) -> str: def get_executor_pnl(executor: Dict[str, Any]) -> float: """Extract PnL from an executor response.""" for key in ( - "net_pnl_quote", "pnl_quote", "unrealized_pnl_quote", - "realized_pnl_quote", "net_pnl", "pnl", "close_pnl", + "net_pnl_quote", + "pnl_quote", + "unrealized_pnl_quote", + "realized_pnl_quote", + "net_pnl", + "pnl", + "close_pnl", ): val = executor.get(key) if val is not None and val != 0: diff --git a/condor/fetchers/market_data.py b/condor/fetchers/market_data.py index fd3de269d..48da5b69f 100644 --- a/condor/fetchers/market_data.py +++ b/condor/fetchers/market_data.py @@ -40,8 +40,7 @@ async def fetch_candles( ) if not candles: return None - data = candles if isinstance(candles, list) else candles.get("data", []) - if not data: + if not _unwrap_candles(candles): return None return candles except Exception as e: @@ -51,6 +50,156 @@ async def fetch_candles( return None +def normalize_candle(c: Any, *, strict: bool = False) -> Optional[Dict[str, float]]: + """Normalize one candle row to a dict of floats, or None when unusable. + + The API returns candles in two shapes depending on endpoint and version: a + dict keyed by OHLCV names, or a positional row ``[ts, o, h, l, c, v]`` + (extra trailing fields are ignored). Rows in neither shape yield None. + + Args: + strict: Re-raise ``TypeError``/``ValueError`` from a malformed value + (e.g. ``open="n/a"``) instead of dropping the row. No candle caller + asks for this today — every one of them would rather lose a candle + than a whole series (see CORR-168) — but the raise stays available + for a consumer that cannot tolerate a hole. + """ + try: + if isinstance(c, dict): + return { + "timestamp": float(c.get("timestamp", 0)), + "open": float(c.get("open", 0)), + "high": float(c.get("high", 0)), + "low": float(c.get("low", 0)), + "close": float(c.get("close", 0)), + "volume": float(c.get("volume", 0)), + } + if isinstance(c, (list, tuple)) and len(c) >= 6: + return { + "timestamp": float(c[0]), + "open": float(c[1]), + "high": float(c[2]), + "low": float(c[3]), + "close": float(c[4]), + "volume": float(c[5]), + } + except (TypeError, ValueError): + if strict: + raise + return None + + +def _unwrap_candles(result: Any) -> List[Any]: + """Candle rows out of either payload shape: a bare list or ``{"data": [...]}``.""" + return ( + result + if isinstance(result, list) + else result.get("data", []) if isinstance(result, dict) else [] + ) + + +async def fetch_historical_candles( + client, + connector_name: str, + trading_pair: str, + interval: str = "1m", + *, + start_time: Optional[int] = None, + end_time: Optional[int] = None, + limit: Optional[int] = None, + fallback_on_error: bool = False, + strict: bool = False, +) -> List[Dict[str, float]]: + """Time-ranged candles, normalized, with the ``get_candles`` fallback ladder. + + Owns the sequence every candle consumer used to inline: ask for the historical + range, unwrap whichever payload shape came back, fall back to the plain + ``get_candles`` window when the range yields nothing, and normalize the rows. + + Args: + start_time / end_time: Unix epoch seconds. With ``start_time`` None the + historical call is skipped entirely and the fallback (if any) answers. + limit: Row count for the ``get_candles`` fallback. None disables the + fallback — for pollers that only want the fresh tail of a live range + and would rather return nothing than a full unrelated window. + fallback_on_error: Treat a failing historical call as an empty result and + continue to the fallback, instead of propagating. Callers that abort + on failure (WS backfill, REST poll) leave it off. + strict: Passed through to `normalize_candle`. Rows dropped instead of + raised are logged once per call at warning, so a payload bug is + still discoverable rather than silently thinning the series. + + Returns: + Normalized candles in upstream order; empty when nothing was usable. + """ + result = None + if start_time is not None: + try: + logger.info( + "Fetching historical candles: connector=%s pair=%s interval=%s " + "start=%s end=%s", + connector_name, + trading_pair, + interval, + start_time, + end_time, + ) + result = await client.market_data.get_historical_candles( + connector_name, + trading_pair, + interval, + start_time=start_time, + end_time=end_time, + ) + except Exception as e: + if not fallback_on_error: + raise + logger.warning( + "get_historical_candles failed: %s — falling back to get_candles", e + ) + result = None + + rows = _unwrap_candles(result) + if not rows and limit is not None: + logger.info( + "Falling back to get_candles: connector=%s pair=%s interval=%s limit=%s", + connector_name, + trading_pair, + interval, + limit, + ) + result = await client.market_data.get_candles( + connector_name, trading_pair, interval, limit + ) + rows = _unwrap_candles(result) + + normalized: List[Dict[str, float]] = [] + dropped: List[Any] = [] + for r in rows: + c = normalize_candle(r, strict=strict) + if c is None: + dropped.append(r) + else: + normalized.append(c) + + if dropped: + # A dropped row leaves a hole in the series that consumers cannot tell + # apart from a genuine gap in upstream data, so keep it discoverable. + # One line per fetch, not per bad row: a systematically broken payload + # would otherwise flood the log from the WS pollers. + logger.warning( + "Dropped %d of %d unusable candle rows for %s %s %s; first: %r", + len(dropped), + len(rows), + connector_name, + trading_pair, + interval, + dropped[0], + ) + + return normalized + + async def fetch_candle_connectors(client, **_kw) -> List[str]: """Fetch available candle connectors.""" return await client.market_data.get_available_candle_connectors() diff --git a/condor/fetchers/portfolio.py b/condor/fetchers/portfolio.py index 3d39bcac3..7e11ffba1 100644 --- a/condor/fetchers/portfolio.py +++ b/condor/fetchers/portfolio.py @@ -1,18 +1,44 @@ """Fetch portfolio / balance data from Hummingbot API.""" import logging -from typing import Any, Dict, List, Optional +import time +from typing import Any, Dict, List, Optional, Tuple from condor.fetchers.connectors import fetch_available_cex_connectors logger = logging.getLogger(__name__) +# Selectable history windows: range key -> (lookback seconds, candle interval). +# Lives here rather than in the REST route because both the route and the WS +# manager (which subscribes one SDS key per range) need it, and neither layer +# may import the other. +PORTFOLIO_HISTORY_RANGES: Dict[str, Tuple[int, str]] = { + "1D": (86400, "5m"), + "1W": (604800, "1h"), + "1M": (2592000, "4h"), + "3M": (7776000, "1d"), +} + +# The poll cadence is not declared here: each range is polled at its own TTL, +# which ``ServerDataType.PORTFOLIO_HISTORY``'s defaults already state (see +# ``DataTypeDefaults.interval_for``). A second table here would only be one +# more thing that has to agree with that one. + async def fetch_portfolio(client, **_kw) -> Any: """Fetch full portfolio state from a server.""" return await client.portfolio.get_state() +async def fetch_portfolio_history(client, range_key: str = "1D", **_kw) -> Any: + """Fetch the raw portfolio history snapshots for one range window.""" + range_seconds, interval = PORTFOLIO_HISTORY_RANGES[range_key] + start_time = int(time.time()) - range_seconds + return await client.portfolio.get_history( + start_time=start_time, interval=interval, limit=500 + ) + + async def fetch_portfolio_refreshed(client, **_kw) -> Any: """Fetch portfolio state with refresh=True to force exchange re-fetch.""" return await client.portfolio.get_state(refresh=True) diff --git a/condor/fetchers/trading_rules.py b/condor/fetchers/trading_rules.py index 809bdb9d5..c10b26af7 100644 --- a/condor/fetchers/trading_rules.py +++ b/condor/fetchers/trading_rules.py @@ -43,11 +43,14 @@ async def fetch_trading_rules( if "404" in error_str or "401" in error_str or "not found" in error_str.lower(): logger.debug( "Connector '%s' not available for trading rules: %s", - connector_name, e, + connector_name, + e, ) else: logger.error( "Error fetching trading rules for %s: %s", - connector_name, e, exc_info=True, + connector_name, + e, + exc_info=True, ) return {} diff --git a/condor/fsutil.py b/condor/fsutil.py new file mode 100644 index 000000000..3577de41f --- /dev/null +++ b/condor/fsutil.py @@ -0,0 +1,143 @@ +"""One atomic file write for the whole codebase. + +Seven independent copies of the "write a temp file, then ``os.replace`` it over +the target" dance used to live in ``condor/`` (memory store, canvas, routine +hooks, code runs, the report store twice, the runtime status registry and the +pickle persistence). They had already drifted apart in ways that mattered: +only one had ``fsync``, only one wrote to a *fixed* temp name (which defeats +the atomicity as soon as two writers race — CORR-032), two wrote in the locale +encoding instead of UTF-8, and two left the temp file visible in a directory +the app lists. ARCH-148 collapses them into this module. + +The implementation here is the **safest superset** of what the copies did: + +* **Unique temp file per writer.** ``tempfile.mkstemp`` opens it ``O_EXCL`` in + the target's own directory, so concurrent writers never share — and thus + never tear — the temp file, and ``os.replace`` stays a same-filesystem + rename. +* **Hidden, target-derived temp name** (``...tmp``), so a + half-written file is never picked up by a listing of the directory and is + obvious to a human who finds one. +* **Durability.** The data is ``fsync``-ed before the rename and the parent + directory is ``fsync``-ed after it, so a power loss leaves either the old + file or the new one — never a zero-length survivor of the rename. Pass + ``fsync=False`` on a writer hot enough that the barrier costs more than the + data is worth; every current caller keeps it on (an agent tick, a report + save and a config save are all far slower than the flush). +* **UTF-8 always**, so a file written on a machine with a non-UTF-8 locale is + still readable on one that has it. +* **Mode preserved.** ``mkstemp`` creates ``0600``, which would silently + tighten a file that already existed with looser permissions; the target's + current mode is carried over when there is one, and new files stay ``0600``. +* **No temp file left behind**, on success (the rename consumes it) or on any + failure, including ``KeyboardInterrupt``. +""" + +from __future__ import annotations + +import json +import os +import stat +import tempfile +from pathlib import Path +from typing import Any + +__all__ = ["atomic_write_bytes", "atomic_write_text", "atomic_write_json"] + +# Mode for a file this helper creates from scratch. Matches what +# ``tempfile.mkstemp`` produces, i.e. what most of the folded-in copies already +# left on disk: these are the application's own data files, not shared ones. +_NEW_FILE_MODE = 0o600 + + +def atomic_write_bytes( + path: str | Path, data: bytes, *, fsync: bool = True, mkdir: bool = True +) -> None: + """Create-or-replace ``path`` with ``data``, atomically. + + A reader of ``path`` sees either the previous contents or the complete new + ones, never a partial write, and a crash at any point leaves the previous + file intact. + """ + path = Path(path) + if mkdir: + path.parent.mkdir(parents=True, exist_ok=True) + + mode = _target_mode(path) + fd, tmp_name = tempfile.mkstemp( + dir=str(path.parent), prefix=f".{path.name}.", suffix=".tmp" + ) + try: + try: + handle = os.fdopen(fd, "wb") + except BaseException: + os.close(fd) + raise + with handle: + handle.write(data) + if fsync: + handle.flush() + os.fsync(handle.fileno()) + os.chmod(tmp_name, mode) + os.replace(tmp_name, path) + except BaseException: + # Nothing published: drop the temp file rather than leave litter next + # to the real one. + try: + os.unlink(tmp_name) + except OSError: + pass + raise + + if fsync: + _fsync_dir(path.parent) + + +def atomic_write_text( + path: str | Path, + text: str, + *, + encoding: str = "utf-8", + fsync: bool = True, + mkdir: bool = True, +) -> None: + """:func:`atomic_write_bytes` for text. UTF-8 unless told otherwise.""" + atomic_write_bytes(path, text.encode(encoding), fsync=fsync, mkdir=mkdir) + + +def atomic_write_json( + path: str | Path, + data: Any, + *, + fsync: bool = True, + mkdir: bool = True, + **json_kw: Any, +) -> None: + """:func:`atomic_write_text` over ``json.dumps(data, **json_kw)``. + + Serialization happens *before* the temp file is created, so a value that + cannot be encoded raises without having touched the filesystem at all. + """ + atomic_write_text(path, json.dumps(data, **json_kw), fsync=fsync, mkdir=mkdir) + + +def _target_mode(path: Path) -> int: + """Permissions to publish with: the target's own, or ``0600`` if new.""" + try: + return stat.S_IMODE(path.stat().st_mode) + except OSError: + return _NEW_FILE_MODE + + +def _fsync_dir(directory: Path) -> None: + """Flush the rename itself. Best effort: not every filesystem allows it.""" + try: + fd = os.open(str(directory), os.O_RDONLY) + except OSError: + return + try: + os.fsync(fd) + except OSError: + pass + finally: + os.close(fd) diff --git a/condor/memory/store.py b/condor/memory/store.py index db1225cd3..0284824eb 100644 --- a/condor/memory/store.py +++ b/condor/memory/store.py @@ -22,14 +22,14 @@ from __future__ import annotations import json -import os import re -import uuid from datetime import datetime, timezone from pathlib import Path import yaml +from condor.fsutil import atomic_write_text + from .paths import store_root _VALID_TYPES = ("preference", "fact", "feedback", "reference") @@ -109,22 +109,15 @@ def _atomic_write(path: Path, text: str) -> None: Free function (not a method) so every per-assistant store — memories and skills alike — shares one implementation instead of a verbatim copy. The - temp file is named uniquely per writer (pid + uuid) so two processes - writing the same slug concurrently never share — and thus never tear — the - temp file; ``os.replace`` then publishes each writer's complete file - atomically (FEAT-003 runs the same store from the main process and the MCP - subprocess). + temp file is unique per writer, so two processes writing the same slug + concurrently never share — and thus never tear — it; ``os.replace`` then + publishes each writer's complete file atomically (FEAT-003 runs the same + store from the main process and the MCP subprocess). + + Thin alias over :func:`condor.fsutil.atomic_write_text`, which is the one + implementation of this dance in the codebase (ARCH-148). """ - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_name(f".{path.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp") - try: - tmp.write_text(text) - os.replace(tmp, path) - finally: - try: - tmp.unlink() - except FileNotFoundError: - pass + atomic_write_text(path, text) def _trim_audit(audit_file: Path, cap: int | None = None) -> None: @@ -146,17 +139,7 @@ def _trim_audit(audit_file: Path, cap: int | None = None) -> None: if len(lines) <= cap * 2: return tail = lines[-cap:] - tmp = audit_file.with_name( - f".{audit_file.name}.{os.getpid()}.{uuid.uuid4().hex}.tmp" - ) - try: - tmp.write_text("".join(tail)) - os.replace(tmp, audit_file) - finally: - try: - tmp.unlink() - except FileNotFoundError: - pass + _atomic_write(audit_file, "".join(tail)) class MemoryStore: diff --git a/condor/orca_api.py b/condor/orca_api.py new file mode 100644 index 000000000..8e48022d4 --- /dev/null +++ b/condor/orca_api.py @@ -0,0 +1,121 @@ +"""Orca's public whirlpool API — the single module that talks to api.orca.so. + +Gateway can already list Orca's pools, but only as a thin proxy over *this* API +that drops almost everything on the way: it answers TVL, fee tier and tick spacing +and nulls `volume_24h`, `fees_24h`, `apr`, `apy` and `current_price`, so the browser +had to make a second trip to GeckoTerminal for volume and then *estimate* the fee +yield from volume times the base tier. It also ignores its `page` argument for this +connector, so every page of the Orca tab was page one. + +Reading Orca directly answers all of it in one request, with real fees (rewards +included) rather than an estimate, and with pagination that works. + +Public and unauthenticated, but Cloudflare-fronted: a request with no or a +default-library User-Agent is answered 403, so one is always sent. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import Any, Dict, List, Optional + +import httpx + +logger = logging.getLogger(__name__) + +ORCA_API_BASE = "https://api.orca.so/v2/solana" + +# `Python-urllib/x` is 403'd by Cloudflare outright. Identifying the caller is the +# courteous thing to do against a free public API in any case. +_USER_AGENT = "condor-dex-browser/1.0 (+https://github.com/hummingbot)" + +# The browser is waiting on this: a slow listing is better abandoned and served +# from the stale cache than held open. +_TIMEOUT = httpx.Timeout(15.0, connect=5.0) + +# One client, and so one connection pool — the same reason ``pool_data`` keeps a +# single GeckoTerminal client. Kept with the loop it was built on: a pooled +# connection belongs to that loop, and reusing it from another one fails with +# "Event loop is closed" rather than reconnecting. The web app is one long-lived +# loop, but a routine or a test driving this through ``asyncio.run`` is not. +_client: Optional[httpx.AsyncClient] = None +_client_loop: Optional[asyncio.AbstractEventLoop] = None + + +def _http() -> httpx.AsyncClient: + global _client, _client_loop + loop = asyncio.get_running_loop() + if _client is None or _client.is_closed or _client_loop is not loop: + # The previous client's connections died with its loop; there is nothing + # left to close, so the reference is simply dropped. + _client = httpx.AsyncClient( + base_url=ORCA_API_BASE, + timeout=_TIMEOUT, + headers={"User-Agent": _USER_AGENT, "Accept": "application/json"}, + ) + _client_loop = loop + return _client + + +async def _get(path: str, params: Dict[str, Any]) -> Any: + response = await _http().get(path, params=params) + response.raise_for_status() + return response.json() + + +def _rows(payload: Any) -> List[Dict[str, Any]]: + """The whirlpool rows out of Orca's ``{"data": [...], "meta": {...}}`` envelope.""" + data = payload.get("data") if isinstance(payload, dict) else payload + if isinstance(data, dict): # a single-pool response + data = [data] + if not isinstance(data, list): + return [] + return [row for row in data if isinstance(row, dict)] + + +async def fetch_whirlpools( + size: int, + search: Optional[str] = None, + sort_by: str = "tvl", + sort_direction: str = "desc", + stats: str = "24h", +) -> List[Dict[str, Any]]: + """Up to ``size`` whirlpools, ranked, as raw Orca rows. + + One request rather than a cursor walk. Orca lists over 8,000 whirlpools but + fewer than ~500 of them hold more than $10k, and ranking by TVL puts those + first — so a single page covers everything the browser would page through, + which is also what lets the caller keep offset pagination over a cursor-only + upstream. + + ``stats`` picks which time windows to hydrate; only the ones actually rendered + are asked for, since the default hydrates 24h, 7d and 30d for every row. + + Raises on an upstream failure rather than returning ``[]``: a 429 and a chain + with no pools are different answers, and only the caller knows whether it has + a stale copy to fall back on. + """ + params: Dict[str, Any] = { + "size": max(1, int(size)), + "sortBy": sort_by, + "sortDirection": sort_direction, + "stats": stats, + } + search = (search or "").strip() + if search: + # A different endpoint, because /pools filters by mint and this matches + # free text against symbols and addresses — which is what the tab's search + # box offers. + params["q"] = search + return _rows(await _get("/pools/search", params)) + return _rows(await _get("/pools", params)) + + +async def close() -> None: + """Release the connection pool. For tests and shutdown.""" + global _client, _client_loop + if _client is not None and not _client.is_closed: + await _client.aclose() + _client = None + _client_loop = None diff --git a/condor/persistence.py b/condor/persistence.py index bf023c214..d9e242d3e 100644 --- a/condor/persistence.py +++ b/condor/persistence.py @@ -10,10 +10,10 @@ to prevent pickle bloat. """ +import io import logging import os import pickle -import tempfile from copy import deepcopy from pathlib import Path from typing import Any, Dict, Optional @@ -21,6 +21,8 @@ from telegram.ext import PicklePersistence from telegram.ext._picklepersistence import _BotPickler, _BotUnpickler +from condor.fsutil import atomic_write_bytes + logger = logging.getLogger(__name__) # Keys that live in user_data at runtime but should NOT be persisted. @@ -60,20 +62,12 @@ def _dump_singlefile(self) -> None: } target = self.filepath - tmp_fd = None - tmp_path = None try: - # 1. Write to a temp file in the same directory - tmp_fd, tmp_path = tempfile.mkstemp( - suffix=".tmp", - dir=str(target.parent), - ) - with os.fdopen(tmp_fd, "wb") as f: - tmp_fd = None # os.fdopen takes ownership - _BotPickler(self.bot, f, protocol=pickle.HIGHEST_PROTOCOL).dump(data) - f.flush() - os.fsync(f.fileno()) + # 1. Serialize first: a pickling failure must not cost us the + # backup rotation below, let alone the current file. + buf = io.BytesIO() + _BotPickler(self.bot, buf, protocol=pickle.HIGHEST_PROTOCOL).dump(data) # 2. Rotate current file to .bak (best-effort) bak_path = target.with_suffix(target.suffix + ".bak") @@ -83,22 +77,12 @@ def _dump_singlefile(self) -> None: except OSError: logger.warning("Failed to create backup of pickle file") - # 3. Atomic rename of temp → target - os.replace(tmp_path, str(target)) - tmp_path = None # success – nothing to clean up + # 3. Atomic temp-file → fsync → rename (shared helper, ARCH-148) + atomic_write_bytes(target, buf.getvalue()) except Exception: logger.exception("Failed to write persistence file") raise - finally: - # Clean up temp file on failure - if tmp_fd is not None: - os.close(tmp_fd) - if tmp_path is not None: - try: - os.unlink(tmp_path) - except OSError: - pass # ------------------------------------------------------------------ # Load: fall back to .bak on corruption @@ -124,9 +108,7 @@ def _load_singlefile(self) -> None: self.conversations = data.get("conversations", {}) self.user_data = data.get("user_data", {}) self.chat_data = data.get("chat_data", {}) - self.bot_data = data.get( - "bot_data", self.context_types.bot_data() - ) + self.bot_data = data.get("bot_data", self.context_types.bot_data()) self.callback_data = data.get("callback_data", None) else: # Both files missing or corrupt – start fresh diff --git a/condor/pool_data.py b/condor/pool_data.py new file mode 100644 index 000000000..cb2cf3796 --- /dev/null +++ b/condor/pool_data.py @@ -0,0 +1,2292 @@ +""" +Pool Data Utilities + +Provides unified data fetching for DEX pools: +- OHLCV data via GeckoTerminal (works for any pool on any DEX) +- Token symbol / top-pool resolution via GeckoTerminal +- Liquidity/bin data via Gateway CLMM (for supported DEXes) +- Pool info normalization across different sources + +This is the single module that talks to GeckoTerminal for pool and token market +data. Consumers (Telegram handlers, the web dashboard's market routes) call these +functions rather than building GeckoTerminal URLs themselves. +""" + +import asyncio +import logging +import math +import re +import time +from collections import deque +from typing import Any, Dict, List, Optional, Tuple + +import httpx +from geckoterminal_py import GeckoTerminalAsyncClient +from geckoterminal_py import constants as GECKO_CONSTANTS +from glom import glom + +from condor import orca_api +from condor.cache import evict_expired, get_cached, set_cached +from config_manager import get_client + +logger = logging.getLogger(__name__) + +# The conversation cache these functions write into is shared with the Telegram DEX +# handlers, which read and invalidate it by group under the "_cache" namespace (see +# handlers/dex/_shared). Entries written under any other namespace would survive an +# invalidation the handler thinks it performed, so this must stay in step with it. +_CACHE_NS = "_cache" + +# CLMM venues an ``lp_executor`` can open a position on: (brand, the qualifiers its +# GeckoTerminal dex id must carry, the gecko networks it exists on). Deliberately +# separate from LIQUIDITY_SUPPORTED_DEXES below, which answers a different question +# (can Condor draw Telegram's liquidity-bin chart) and compares gecko network ids +# against chain *names*, so it is false for every Uniswap pool on Ethereum. +_CLMM_VENUES: Tuple[Tuple[str, List[str], set], ...] = ( + # Gecko's `meteora` is DLMM and `orca` is Whirlpools — both already the + # concentrated product, so any qualifier means a different pool type. + ("meteora", [], {"solana"}), + ("orca", [], {"solana"}), + # Plain `raydium` is the constant-product AMM v4; only `raydium-clmm` is CLMM. + ("raydium", ["clmm"], {"solana"}), + ("uniswap", ["v3"], {"eth", "arbitrum", "base", "polygon_pos", "optimism", "bsc"}), + ("pancakeswap", ["v3"], {"bsc", "eth", "base", "arbitrum"}), +) + +# Chain suffixes gecko tacks onto a dex id (`uniswap-v3-base`). They say where the +# venue is, not which product it is, so they are dropped before matching. +_CHAIN_TOKENS = { + "solana", + "ethereum", + "eth", + "base", + "arbitrum", + "bsc", + "polygon", + "pos", + "optimism", + "avalanche", +} + +_DEX_ID_SPLIT_RE = re.compile(r"[-_]") + +# Supported DEXes for liquidity data (via gateway CLMM) +LIQUIDITY_SUPPORTED_DEXES = { + "meteora": "solana", + "raydium": "solana", + "orca": "solana", + "uniswap": "ethereum", + "pancakeswap": "bsc", +} + +# GeckoTerminal network mapping. +# +# The keys are the network ids Gateway actually reports from ``/gateway/networks`` +# — ``chain-network``, so every EVM chain is ``ethereum-``. That is not +# cosmetic: the ids this table used to carry (``base-mainnet``, ``arbitrum-one``, +# ``polygon-mainnet``) are ones Gateway never emits, so every EVM pool fell through +# to "no gecko chain" and rendered as untradable. The older spellings are kept as +# aliases because persisted preferences and Telegram flows still hold them. +NETWORK_TO_GECKO = { + # Solana + "solana": "solana", + "solana-mainnet-beta": "solana", + # EVM, as Gateway names them + "ethereum": "eth", + "ethereum-mainnet": "eth", + "ethereum-arbitrum": "arbitrum", + "ethereum-avalanche": "avax", + "ethereum-base": "base", + "ethereum-bsc": "bsc", + "ethereum-celo": "celo", + "ethereum-optimism": "optimism", + "ethereum-polygon": "polygon_pos", + "ethereum-robinhoodchain": "robinhood", + "ethereum-unichain": "unichain", + # Legacy spellings still held by persisted preferences and Telegram flows. + "arbitrum": "arbitrum", + "arbitrum-one": "arbitrum", + "base": "base", + "base-mainnet": "base", + "bsc": "bsc", + "binance-smart-chain": "bsc", + "polygon": "polygon_pos", + "polygon-mainnet": "polygon_pos", + # Gecko's id for Avalanche is `avax`, not `avalanche` — the old value here + # matched no chain at all. + "avalanche": "avax", + "avalanche-mainnet": "avax", + "optimism": "optimism", + # Not a Gateway network: `xrpl` is a first-class Hummingbot connector (orders, + # balances, order book) that CandlesFactory simply has no feed for. Its AMM + # pools are indexed by GeckoTerminal, so its charts resolve here like a DEX's. + "xrpl": "xrpl", +} + +# The inverse of NETWORK_TO_GECKO, one entry per gecko chain: the Gateway network +# id a pool discovered on that chain belongs to. Lives here rather than in the +# Telegram handler that used to own it because the pool browser's rows are routed +# by it (``/dex/{gateway_network}/{address}``), and a gecko chain with no entry is +# one Gateway cannot reach — a state to render, not an error. +# +# Every value must be a key of NETWORK_TO_GECKO, because `decorate_pool` decides +# `tradable` by looking the result up there; a value that round-trips to nothing +# marks a perfectly reachable chain untradable. +GECKO_TO_GATEWAY_NETWORK = { + "solana": "solana-mainnet-beta", + "eth": "ethereum-mainnet", + "arbitrum": "ethereum-arbitrum", + "avax": "ethereum-avalanche", + "base": "ethereum-base", + "bsc": "ethereum-bsc", + "celo": "ethereum-celo", + "optimism": "ethereum-optimism", + "polygon_pos": "ethereum-polygon", + "robinhood": "ethereum-robinhoodchain", + "unichain": "ethereum-unichain", +} + +# Networks whose venues quote *tickers* rather than token addresses. An XRPL token +# is a (currency, issuer) pair that the connector hides behind a symbol like +# ``SOLO-XRP``, so its pool can only be found by searching for that symbol — +# unlike a Solana pair, whose base already *is* the mint. +SYMBOL_PAIR_NETWORKS = {"xrpl"} + +# DEX ID to GeckoTerminal DEX mapping +DEX_TO_GECKO = { + "meteora": "meteora", + "raydium": "raydium", + "orca": "orca", + "uniswap": "uniswap", + "uniswap_v3": "uniswap_v3", + "pancakeswap": "pancakeswap", + "pancakeswap_v3": "pancakeswap_v3", + "sushiswap": "sushiswap", +} + +# Cache TTLs +OHLCV_CACHE_TTL = 300 # 5 minutes +BINS_CACHE_TTL = 60 # 1 minute +TOKEN_SYMBOL_TTL = 24 * 3600 # a mint's ticker does not change +TOKEN_POOL_TTL = 3600 # a token's main pool is stable over an hour +# Pool *lists* are the pool browser's hot path and GeckoTerminal rate-limits per +# IP across every dashboard viewer and every polling chart, so these are not an +# optimization: an un-cached browser refetching per keystroke would starve the +# candle poll loop of the same budget. +POOL_LIST_TTL = 60 # gecko trending/top/new/token lists +GATEWAY_POOL_LIST_TTL = 30 # gateway CLMM lists, per (connector, search) +# One Orca snapshot serves every page and every column sort of the Orca tab, so +# this is a whole tab's upstream budget rather than one page's — cheaper than the +# Gateway path it replaces, which spent a request per page and a gecko one on top. +ORCA_POOL_LIST_TTL = 60 + +# GeckoTerminal's OHLCV endpoint only accepts these aggregates, and its client +# raises ValueError on anything else. Charts pick their own interval, so map an +# unsupported one down to the nearest supported bucket rather than returning an +# empty chart. +GECKO_TIMEFRAMES = ("1m", "5m", "15m", "1h", "4h", "12h", "1d") +_TIMEFRAME_SECONDS = { + "1m": 60, + "5m": 300, + "15m": 900, + "1h": 3600, + "4h": 14400, + "12h": 43200, + "1d": 86400, +} +# GeckoTerminal caps a single OHLCV response at this many candles. +GECKO_OHLCV_MAX = 1000 + + +def get_gecko_network(network: str) -> str: + """Convert internal network name to GeckoTerminal network ID""" + return NETWORK_TO_GECKO.get(network, network) + + +def uses_symbol_pairs(network: str) -> bool: + """Whether this network's trading pairs are tickers instead of addresses.""" + return get_gecko_network(network) in SYMBOL_PAIR_NETWORKS + + +def timeframe_seconds(timeframe: str) -> int: + """Length of one candle in seconds; 60 for anything unrecognized.""" + return _TIMEFRAME_SECONDS.get(timeframe, 60) + + +def normalize_timeframe(interval: str) -> str: + """Snap an arbitrary chart interval onto a GeckoTerminal timeframe. + + Exact matches pass through. Anything else (``3m``, ``30m``, ``1s``…) resolves + to the largest supported timeframe that is no coarser than requested, so the + chart keeps at least the resolution it asked for. Unparseable input falls back + to ``1m``. + """ + if interval in _TIMEFRAME_SECONDS: + return interval + unit_seconds = {"s": 1, "m": 60, "h": 3600, "d": 86400} + try: + wanted = int(interval[:-1]) * unit_seconds[interval[-1]] + except (ValueError, KeyError, IndexError): + return "1m" + supported = sorted(_TIMEFRAME_SECONDS.items(), key=lambda kv: kv[1]) + best = supported[0][0] + for name, seconds in supported: + if seconds <= wanted: + best = name + return best + + +def candles_needed(start: Optional[float], end: Optional[float], timeframe: str) -> int: + """How many candles cover ``[start, end]`` at ``timeframe``. + + The chart asks for a *window*; GeckoTerminal answers with a *count* walking + back from ``before_timestamp``. Without this translation a ten-minute position + charted at 1m would come back with the maximum 1000 candles — ~16h of history + that ``fitContent()`` then squeezes the actual position into. + """ + if start is None or end is None or end <= start: + return GECKO_OHLCV_MAX + span = math.ceil((end - start) / timeframe_seconds(timeframe)) + # A couple of candles of headroom so the window's edges are never clipped. + return max(1, min(span + 2, GECKO_OHLCV_MAX)) + + +# ── GeckoTerminal client ── +# One client (and so one httpx connection pool) for the process. Constructing a +# fresh GeckoTerminalAsyncClient per call leaks an unclosed httpx.AsyncClient +# every time, which at chart-refresh rates exhausts sockets. +_gecko_client_instance: Optional[GeckoTerminalAsyncClient] = None + + +def _gecko_client() -> GeckoTerminalAsyncClient: + global _gecko_client_instance + if _gecko_client_instance is None: + _gecko_client_instance = GeckoTerminalAsyncClient() + # geckoterminal_py builds its httpx client with no timeout override, so a + # slow chain listing blocks for httpx's 5s default and then fails outright. + # A read timeout well above that is worth more than a fast empty table. + try: + _gecko_client_instance._client.timeout = httpx.Timeout( + _GECKO_TIMEOUT, connect=5.0 + ) + except Exception: # pragma: no cover - a library layout change, not an error + logger.debug("could not set gecko client timeout", exc_info=True) + return _gecko_client_instance + + +# GeckoTerminal's free tier is ~30 requests/minute *per IP*, and that budget is +# shared by every dashboard viewer, the candle poll loop and the pool browser at +# once. Three separate guards defend it, because they fail differently: +# +# * ``_gecko_gate`` (concurrency) turns a burst into a queue, so one page load +# with a dex filter cannot open eight sockets at once. +# * ``_acquire_rate_slot`` (rate) is the one that actually keeps us under the +# limit. A concurrency cap alone does not: four *fast* concurrent requests +# sustain well over 100/min, which is how the budget was being spent before +# anything looked like a burst. +# * ``_trip_breaker`` (circuit) stops calling at all for a moment once gecko has +# said 429. Every request sent during a throttled window is spent for nothing +# and keeps the window rolling, so the cheapest response is silence. +_GECKO_MAX_CONCURRENCY = 4 +_GECKO_TIMEOUT = 15.0 +_GECKO_RETRIES = 2 +_gecko_semaphore: Optional[asyncio.Semaphore] = None + +# Held under gecko's ~30 so a burst that straddles the window boundary still +# lands inside it, and so a Telegram handler racing the dashboard has headroom. +_GECKO_RATE_LIMIT = 25 +_GECKO_RATE_WINDOW = 60.0 + +# How long a caller will queue for a slot before giving up. A web request is +# waiting on the other end, and this module's standing answer to "no budget" is +# the stale cache the callers already fall back to — which beats a request that +# hangs for most of a minute and then succeeds into a closed browser tab. +_GECKO_MAX_WAIT = 5.0 + +# Silence after a 429. Long enough for a rolling per-minute window to drain most +# of the way, short enough that a chart recovers on its next poll rather than +# staying blank until someone reloads. +_GECKO_COOLDOWN = 20.0 + + +class GeckoRateLimited(RuntimeError): + """Raised *instead of* calling gecko when the shared budget is spent. + + Carries "rate limit" in its message so ``_is_rate_limited`` recognizes it and + callers that already branch on a throttled upstream need no change. + """ + + +# Timestamps of the requests inside the current rolling window. +_gecko_call_times: deque = deque() +_gecko_rate_lock: Optional[asyncio.Lock] = None + +# Circuit-breaker + health state. Process-wide, because the budget is per-IP: +# one viewer's 429 is every viewer's 429, and there is no point rediscovering it +# once per chart. +_gecko_cooldown_until: float = 0.0 +_gecko_last_429: float = 0.0 +_gecko_429_count: int = 0 +_gecko_throttled_calls: int = 0 + + +def _gecko_gate() -> asyncio.Semaphore: + """The concurrency gate, built lazily so it binds to the running loop.""" + global _gecko_semaphore + if _gecko_semaphore is None: + _gecko_semaphore = asyncio.Semaphore(_GECKO_MAX_CONCURRENCY) + return _gecko_semaphore + + +def _gecko_rate_gate() -> asyncio.Lock: + """The rate gate, built lazily for the same reason as the semaphore.""" + global _gecko_rate_lock + if _gecko_rate_lock is None: + _gecko_rate_lock = asyncio.Lock() + return _gecko_rate_lock + + +def _is_rate_limited(exc: BaseException) -> bool: + if isinstance(exc, GeckoRateLimited): + return True + resp = getattr(exc, "response", None) + if resp is not None and getattr(resp, "status_code", None) == 429: + return True + return "429" in str(exc) or "rate limit" in str(exc).lower() + + +def _trip_breaker() -> None: + """Record a 429 and stop outbound calls for ``_GECKO_COOLDOWN`` seconds.""" + global _gecko_cooldown_until, _gecko_last_429, _gecko_429_count + now = time.time() + _gecko_last_429 = now + _gecko_429_count += 1 + _gecko_cooldown_until = now + _GECKO_COOLDOWN + logger.warning( + "GeckoTerminal rate-limited (429 #%d) — pausing all requests for %.0fs", + _gecko_429_count, + _GECKO_COOLDOWN, + ) + + +async def _acquire_rate_slot() -> None: + """Claim one request from the shared per-minute budget. + + Waits for a slot up to ``_GECKO_MAX_WAIT``; raises ``GeckoRateLimited`` when + the budget is spent for longer than that, or while the breaker is open. The + wait happens under the lock so callers queue in arrival order instead of + stampeding the instant a slot frees. + """ + global _gecko_throttled_calls + deadline = time.time() + _GECKO_MAX_WAIT + async with _gecko_rate_gate(): + while True: + now = time.time() + + if now < _gecko_cooldown_until: + _gecko_throttled_calls += 1 + raise GeckoRateLimited( + f"GeckoTerminal rate limit: cooling down for another " + f"{_gecko_cooldown_until - now:.1f}s" + ) + + while ( + _gecko_call_times and now - _gecko_call_times[0] >= _GECKO_RATE_WINDOW + ): + _gecko_call_times.popleft() + + if len(_gecko_call_times) < _GECKO_RATE_LIMIT: + _gecko_call_times.append(now) + return + + # Budget full: wait exactly until the oldest request ages out. + wait = _GECKO_RATE_WINDOW - (now - _gecko_call_times[0]) + if now + wait > deadline: + _gecko_throttled_calls += 1 + raise GeckoRateLimited( + f"GeckoTerminal rate limit: {_GECKO_RATE_LIMIT} requests " + f"already sent this minute, next slot in {wait:.1f}s" + ) + await asyncio.sleep(min(wait, 0.25)) + + +def gecko_health() -> Dict[str, Any]: + """Current state of the shared GeckoTerminal budget. + + Exists so "are we rate limited?" has an answer that is not "read the logs" — + every failure downstream of here degrades to an empty list, which cannot tell + a throttled upstream apart from a pool that simply has no trades. + """ + now = time.time() + recent = sum(1 for t in _gecko_call_times if now - t < _GECKO_RATE_WINDOW) + return { + "rate_limited": now < _gecko_cooldown_until, + "cooldown_remaining": max(0.0, _gecko_cooldown_until - now), + "requests_last_minute": recent, + "budget": _GECKO_RATE_LIMIT, + "count_429": _gecko_429_count, + "last_429_age": (now - _gecko_last_429) if _gecko_last_429 else None, + "throttled_calls": _gecko_throttled_calls, + } + + +def reset_gecko_throttle() -> None: + """Forget the spent budget and any open breaker. + + For tests, which drive hundreds of calls through this module inside one real + wall-clock minute and would otherwise throttle themselves. + """ + global _gecko_cooldown_until, _gecko_last_429, _gecko_429_count + global _gecko_throttled_calls + _gecko_call_times.clear() + _gecko_inflight.clear() + _gecko_cooldown_until = 0.0 + _gecko_last_429 = 0.0 + _gecko_429_count = 0 + _gecko_throttled_calls = 0 + for cache in ( + _gecko_page_cache, + _pool_by_address_cache, + _multi_pool_cache, + _gecko_dexes_cache, + _gateway_pool_list_cache, + _token_symbol_cache, + _token_pool_cache, + _pair_pool_cache, + ): + cache.clear() + + +async def gecko_call(method: str, *args: Any, **kwargs: Any) -> Any: + """One GeckoTerminal client call, rate-gated and retried where that helps. + + ``method`` names a method on the shared client. Every outbound GeckoTerminal + request in this module goes through here — calling the client directly skips + the budget, which is exactly how the candle poll loop (the highest-frequency + caller in the process) used to exhaust it while the guards sat idle in front + of two pool-list walks. + + Only timeouts and transport errors are retried. A 429 is deliberately *not*: + gecko's limit is a per-minute window, so no backoff short enough to sit inside + a web request will clear it, and a retry only spends more of the very budget + that is exhausted. It trips the breaker instead. A 404, or the 401 gecko + answers past page 10, is a final answer and is raised on the first attempt. + """ + for attempt in range(_GECKO_RETRIES + 1): + await _acquire_rate_slot() + try: + async with _gecko_gate(): + return await getattr(_gecko_client(), method)(*args, **kwargs) + except Exception as e: # noqa: BLE001 - re-raised below when not retryable + if _is_rate_limited(e): + if not isinstance(e, GeckoRateLimited): + _trip_breaker() + raise + retryable = isinstance(e, (httpx.TimeoutException, httpx.TransportError)) + if not retryable or attempt == _GECKO_RETRIES: + raise + await asyncio.sleep(_GECKO_BACKOFF[attempt]) + raise RuntimeError("unreachable") + + +async def gecko_request(method: str, path: str, **kwargs) -> Any: + """A raw REST call for paths the typed client does not expose.""" + return await gecko_call("api_request", method, path, **kwargs) + + +# Backoff between retries of a *timeout*, which is a slow chain listing rather than +# a throttled one — short, because a web request is waiting on it. +_GECKO_BACKOFF = (0.4, 1.0) + + +# ── Single-flight ── +# A TTL cache only helps once an answer has *arrived*. Flicking between explorer +# tabs asks for the same listings again while the first requests are still in the +# air, so every one of them misses the cache and opens its own gecko request — the +# fastest way there is to spend the minute's budget. This collapses concurrent +# callers of the same key onto one upstream request. +_gecko_inflight: Dict[Tuple, "asyncio.Task"] = {} + + +async def _single_flight(key: Tuple, factory) -> Any: + """Run ``factory()`` once per key, sharing its result with every concurrent caller. + + The work runs as a detached task and awaiters ``shield`` it, so the very thing + that causes the stampede — a viewer navigating away and cancelling their + request — cannot also cancel the fetch the remaining viewers are waiting on. + """ + task = _gecko_inflight.get(key) + if task is None or task.done(): + task = asyncio.ensure_future(factory()) + _gecko_inflight[key] = task + + def _clear(finished: "asyncio.Task", _key: Tuple = key) -> None: + if _gecko_inflight.get(_key) is finished: + _gecko_inflight.pop(_key, None) + + task.add_done_callback(_clear) + return await asyncio.shield(task) + + +# ── Small TTL caches for token lookups ── +# These answers are process-wide (not per-user) and tiny, so they live here rather +# than in a caller's user_data. Capped so an unbounded stream of distinct mints +# cannot grow them without limit. +_TOKEN_CACHE_MAX = 512 +_token_symbol_cache: Dict[Tuple[str, str], Tuple[float, str]] = {} +# Both hold normalized pool dicts, and an empty dict for "asked, there is no such +# pool" — a real answer worth caching, unlike a failed lookup, which is not cached +# at all. The address-only forms (fetch_token_top_pool, fetch_pair_top_pool) read +# these same entries. +_token_pool_cache: Dict[Tuple[str, str, str], Tuple[float, Dict[str, Any]]] = {} +_pair_pool_cache: Dict[ + Tuple[str, str, str], Tuple[float, Tuple[Dict[str, Any], bool]] +] = {} + + +def _ttl_get(cache: dict, key: tuple, ttl: float) -> Optional[Any]: + entry = cache.get(key) + if entry and (time.time() - entry[0]) < ttl: + return entry[1] + return None + + +def _ttl_put(cache: dict, key: tuple, value: Any, ttl: float) -> None: + now = time.time() + for k in [k for k, (ts, _) in cache.items() if now - ts >= ttl]: + cache.pop(k, None) + cache[key] = (now, value) + while len(cache) > _TOKEN_CACHE_MAX: + cache.pop(next(iter(cache))) + + +# ── Stale-tolerant cache, for upstreams that fail more often than they change ── +# The rule everywhere else — never cache a failure — is right, but on its own it +# turns every rate-limited request into an empty table: the browser asks, gecko +# says 429, and the user sees "No pools found" for a chain with thousands. Pool +# listings are ranked-by-volume snapshots, so a two-minute-old copy is a far better +# answer than no answer, and these two helpers exist so a failed refresh can fall +# back to one. +# +# Separate from _ttl_put because that one *sweeps* expired entries on every write, +# which would throw away the very copies this fallback depends on. +_STALE_MAX_AGE = 15 * 60 + + +def _stale_put(cache: dict, key: tuple, value: Any) -> None: + cache[key] = (time.time(), value) + now = time.time() + for k in [k for k, (ts, _) in cache.items() if now - ts >= _STALE_MAX_AGE]: + cache.pop(k, None) + while len(cache) > _TOKEN_CACHE_MAX: + cache.pop(next(iter(cache))) + + +def _stale_get(cache: dict, key: tuple) -> Tuple[Optional[Any], float]: + """The cached value regardless of freshness, with its age in seconds.""" + entry = cache.get(key) + if not entry: + return None, 0.0 + return entry[1], time.time() - entry[0] + + +async def fetch_token_symbol(mint: str, network: str) -> str: + """Resolve a token mint/contract address to its ticker (e.g. ``Bonk``). + + Returns "" when the token is unknown or the lookup fails. An empty answer from + a *successful* response is cached (a genuinely unlisted mint should not be + re-queried on every render); a failed lookup is not, so one blip does not blank + the ticker for a day. + """ + gnet = get_gecko_network(network) + key = (gnet, mint) + cached = _ttl_get(_token_symbol_cache, key, TOKEN_SYMBOL_TTL) + if cached is not None: + return cached + + try: + data = await gecko_call("get_specific_token_on_network", gnet, mint) + except Exception as e: + logger.info("token symbol lookup failed mint=%s net=%s: %s", mint, gnet, e) + return "" + + symbol = "" + if isinstance(data, dict): + symbol = str((data.get("attributes") or {}).get("symbol") or "") + _ttl_put(_token_symbol_cache, key, symbol, TOKEN_SYMBOL_TTL) + return symbol + + +def _pool_quote_symbols(name: Any) -> List[str]: + """Token symbols in a GeckoTerminal pool name (``"BONK / SOL"`` → BONK, SOL).""" + return [p.strip().upper() for p in str(name or "").split("/") if p.strip()] + + +async def fetch_token_top_pool_info( + mint: str, network: str, quote: str +) -> Optional[Dict[str, Any]]: + """The token's highest-volume pool **quoted in ``quote``**, normalized. + + The quote match is not cosmetic: candles are requested with ``currency="token"``, + so prices come back denominated in the pool's quote token. Charting a token/USDC + pool underneath a token/SOL position would silently draw the right shape on the + wrong scale, so a pool that does not match yields ``None`` and the chart stays + empty instead. + + ``None`` distinguishes "no answer yet" from "asked and there is none": a + no-match *is* cached (as ``{}``), a failed lookup is not, so a GeckoTerminal + blip cannot pin an empty answer for the cache's full hour. + """ + gnet = get_gecko_network(network) + want = (quote or "").strip().upper() + key = (gnet, mint, want) + cached = _ttl_get(_token_pool_cache, key, TOKEN_POOL_TTL) + if cached is not None: + return cached or None + + try: + pools = await gecko_call("get_top_pools_by_network_token", gnet, mint) + except Exception as e: + # Includes the KeyError geckoterminal_py raises post-processing an empty + # result set. Not cached — a transient failure must not pin "" for an hour. + logger.info("top-pool lookup failed mint=%s net=%s: %s", mint, gnet, e) + return None + + info: Dict[str, Any] = {} + try: + # Rows arrive sorted by 24h volume, so the first quote match is the deepest. + for row in pools.to_dict("records"): + if want and want not in _pool_quote_symbols(row.get("name")): + continue + if not str(row.get("address") or ""): + continue + info = normalize_pool_data(row, source="gecko") + _fill_pool_pair_fields(info, network, row) + break + except Exception as e: + logger.info("top-pool parse failed mint=%s net=%s: %s", mint, gnet, e) + return None + + _ttl_put(_token_pool_cache, key, info, TOKEN_POOL_TTL) + return info or None + + +async def fetch_token_top_pool(mint: str, network: str, quote: str) -> str: + """Address of the token's highest-volume pool **quoted in ``quote``**. + + Used as a fallback when an executor's own ``pool_address`` yields no candles + (a closed slot, a pool that never had one recorded). The address form of + :func:`fetch_token_top_pool_info`, which is all the candle path needs; the LP + panel wants the venue and price too and shares this function's cache entry. + """ + info = await fetch_token_top_pool_info(mint, network, quote) + return str((info or {}).get("address") or "") + + +async def fetch_pair_top_pool_info( + base: str, quote: str, network: str +) -> Tuple[Optional[Dict[str, Any]], bool]: + """Deepest pool trading ``base``/``quote`` on ``network``, found by *symbol*. + + For venues that quote tickers rather than addresses (see + ``SYMBOL_PAIR_NETWORKS``): an xrpl pair like ``SOLO-XRP`` names no token that + GeckoTerminal can look up, so this searches pools instead and keeps only exact + symbol matches. Ticker collisions are real on XRPL — any issuer can mint a + ``RLUSD`` — so the highest 24h volume wins, which is also the pool the + GeckoTerminal UI shows first. + + Returns: + ``(pool_info, inverted)``. ``inverted`` is True when the pool is quoted + the other way round (pair ``XRP-RLUSD`` against a ``RLUSD / XRP`` pool); + the caller must then read the quote token's price series, not the base's, + and ``pool_info``'s pair fields are reported as the *caller* asked for + them, not as the pool names them. ``(None, False)`` when nothing matches — + an empty chart beats a chart drawn on the wrong pair. + """ + gnet = get_gecko_network(network) + b, q = (base or "").strip().upper(), (quote or "").strip().upper() + if not b or not q: + return None, False + + key = (gnet, b, q) + cached = _ttl_get(_pair_pool_cache, key, TOKEN_POOL_TTL) + if cached is not None: + return (cached[0] or None), cached[1] + + try: + # Both symbols in the query: GeckoTerminal matches them against the pool + # name ("SOLO / XRP"), so this narrows the answer set before we filter. + data = await gecko_request( + "GET", "search/pools", params={"query": f"{b} {q}", "network": gnet} + ) + except Exception as e: + # Not cached — a blip must not pin "no pool" for the full hour. + logger.info("pair-pool search failed %s-%s net=%s: %s", b, q, gnet, e) + return None, False + + best: Tuple[Dict[str, Any], bool] = ({}, False) + best_volume = -1.0 + try: + for row in data.get("data") or []: + attrs = row.get("attributes") or {} + symbols = _pool_quote_symbols(attrs.get("name")) + if len(symbols) != 2: + continue + if symbols == [b, q]: + inverted = False + elif symbols == [q, b]: + inverted = True + else: + continue # fuzzy hit (RLUSDM for RLUSD), not this pair + address = str(attrs.get("address") or "") + if not address: + continue + volume = _get_nested_float(attrs, "volume_usd", "h24") or 0.0 + if volume > best_volume: + flat = _flatten_search_pool(row) + info = normalize_pool_data(flat, source="gecko") + _fill_pool_pair_fields( + info, network, flat["attributes"], inverted=inverted + ) + best, best_volume = (info, inverted), volume + except Exception as e: + logger.info("pair-pool parse failed %s-%s net=%s: %s", b, q, gnet, e) + return None, False + + _ttl_put(_pair_pool_cache, key, best, TOKEN_POOL_TTL) + return (best[0] or None), best[1] + + +async def fetch_pair_top_pool(base: str, quote: str, network: str) -> Tuple[str, bool]: + """``(pool_address, inverted)`` for a symbol-quoted pair. + + The address form of :func:`fetch_pair_top_pool_info`, which is all the candle + path needs; both share the same cache entry. + """ + info, inverted = await fetch_pair_top_pool_info(base, quote, network) + return str((info or {}).get("address") or ""), inverted + + +def _flatten_search_pool(row: Dict[str, Any]) -> Dict[str, Any]: + """Lift a ``search/pools`` row's venue onto its attributes. + + The top-pools *token* endpoint carries ``dex_id`` as a column, but + ``search/pools`` puts the venue in ``relationships.dex.data.id`` instead. The + LP panel's whole question is which venue the pool is on, so the two shapes have + to agree before :func:`normalize_pool_data` sees them. + """ + attrs = dict(row.get("attributes") or {}) + if not attrs.get("dex_id"): + dex = ((row.get("relationships") or {}).get("dex") or {}).get("data") or {} + attrs["dex_id"] = str(dex.get("id") or "unknown") + return {"id": row.get("id", ""), "attributes": attrs} + + +def _fill_pool_pair_fields( + info: Dict[str, Any], + network: str, + attrs: Dict[str, Any], + inverted: bool = False, +) -> None: + """Add the pair fields the LP panel needs, in place. + + GeckoTerminal reports no token *symbol* on a pool row (only the ``"SOL / USDC"`` + display name) and prices in USD or as a quote-token ratio depending on the + endpoint — and ``normalize_pool_data`` keeps neither ratio, hence ``attrs``, the + row it was built from. This fills ``base_symbol`` / ``quote_symbol`` / + ``current_price`` — base priced in quote, the scale every LP bound is expressed + in — and pins ``network`` to what the caller asked for rather than + ``normalize_pool_data``'s ``"solana"`` default. + """ + base_sym, quote_sym = extract_pair_from_name(str(info.get("name") or "")) + ratio = _get_nested_float(attrs, "base_token_price_quote_token") + if ratio is None: + base_usd = _get_nested_float(attrs, "base_token_price_usd") + quote_usd = _get_nested_float(attrs, "quote_token_price_usd") + if base_usd and quote_usd: + ratio = base_usd / quote_usd + + if inverted: + base_sym, quote_sym = quote_sym, base_sym + ratio = (1 / ratio) if ratio else None + + info["base_symbol"] = base_sym.upper() + info["quote_symbol"] = quote_sym.upper() + info["current_price"] = ratio + info["network"] = network + + +async def resolve_pool_info( + network: str, trading_pair: str +) -> Optional[Dict[str, Any]]: + """The pool a trading pair trades in, normalized, or ``None``. + + Mirrors ``condor.dex_candles._resolve_pool``'s branching — a base that *is* a + mint resolves through the token's top pools, a ticker base through a symbol + search — so the LP panel names the same pool the chart is drawn from rather + than resolving pools by a second mechanism. + + It differs in one direction only: ``_resolve_pool`` restricts the symbol + branch to ``SYMBOL_PAIR_NETWORKS``, so a ticker pair like ``SOL-USDC`` on + Solana resolves here but not there (the chart stays blank while the panel + still names a pool). Pointing ``_resolve_pool`` at this function would close + that gap; it is a change to the candle path and deliberately not made here. + """ + # Lazy, like dex_candles' own import of this module: one definition of what an + # on-chain address looks like, and no import cycle between the two. + from condor.dex_candles import ADDRESS_RE + + base, _, quote = str(trading_pair or "").partition("-") + base, quote = base.strip(), quote.strip() + if not base or not quote: + return None + + if ADDRESS_RE.match(base): + return await fetch_token_top_pool_info(base, network, quote) + + info, _inverted = await fetch_pair_top_pool_info(base, quote, network) + return info + + +def network_has_clmm(network: str) -> bool: + """Whether any known CLMM venue exists on this network's gecko chain. + + This is the authority for "can a venue on this network take an ``lp_executor`` + position", and it is deliberately *not* "is this a gateway network". ``xrpl`` is + a gateway-adjacent id that lives in ``NETWORK_TO_GECKO`` for charting only — its + AMM is not concentrated-liquidity, so it answers ``False`` here whether or not + Gateway ever reports it in ``list_networks()``. Deriving the trait from + ``_CLMM_VENUES`` instead of from gateway membership is what keeps that true. + """ + gecko = get_gecko_network(network) + return any(gecko in networks for _brand, _qualifiers, networks in _CLMM_VENUES) + + +def lp_provider_for_dex(dex_id: str, network: str) -> Optional[str]: + """``"meteora/clmm"``-style LP provider for a GeckoTerminal dex id, or ``None``. + + ``None`` means the pool is not one an ``lp_executor`` can add liquidity to, and + the caller is expected to report that dead end rather than guess: the API + rejects an ``lp_provider`` that does not match the pool, so a wrong answer here + becomes a failed executor. + + Matching is by (brand, product) because GeckoTerminal's dex ids are not a + stable vocabulary and brand alone is not the position model. Uniswap V3 is + ``uniswap_v3`` on Ethereum, ``uniswap-v3-base`` on Base and + ``uniswap_v3_arbitrum`` on Arbitrum; ``uniswap-v4-ethereum`` and + ``meteora-damm-v2`` share a brand with a supported venue but not its mechanics; + and plain ``raydium`` is the constant-product AMM, while only ``raydium-clmm`` + is the concentrated pool the ``raydium/clmm`` connector drives. + """ + tokens = [t for t in _DEX_ID_SPLIT_RE.split((dex_id or "").strip().lower()) if t] + if not tokens: + return None + brand, qualifiers = tokens[0], [t for t in tokens[1:] if t not in _CHAIN_TOKENS] + for name, required, networks in _CLMM_VENUES: + if brand != name: + continue + if qualifiers != required: + return None + if get_gecko_network(network) not in networks: + return None + return f"{name}/clmm" + return None + + +def can_fetch_liquidity(dex_id: str, network: str = None) -> bool: + """Check if liquidity/bin data can be fetched for this DEX + + Args: + dex_id: DEX identifier (e.g., "meteora", "raydium") + network: Optional network to verify (must be Solana for now) + + Returns: + True if liquidity data is available via gateway CLMM + """ + dex_lower = dex_id.lower() if dex_id else "" + + if dex_lower not in LIQUIDITY_SUPPORTED_DEXES: + return False + + if network: + expected_network = LIQUIDITY_SUPPORTED_DEXES.get(dex_lower) + gecko_network = get_gecko_network(network) + if gecko_network != expected_network: + return False + + return True + + +def get_connector_for_dex(dex_id: str) -> Optional[str]: + """Get the gateway connector name for a DEX ID + + Args: + dex_id: DEX identifier from GeckoTerminal + + Returns: + Connector name for gateway CLMM or None + """ + dex_lower = dex_id.lower() if dex_id else "" + + # Direct mapping + if dex_lower in LIQUIDITY_SUPPORTED_DEXES: + return dex_lower + + # Handle variations + if "meteora" in dex_lower: + return "meteora" + if "raydium" in dex_lower: + return "raydium" + if "orca" in dex_lower: + return "orca" + + return None + + +async def fetch_ohlcv( + pool_address: str, + network: str, + timeframe: str = "1h", + currency: str = "usd", + user_data: dict = None, + limit: int = 100, + before_timestamp: Optional[int] = None, + token: str = "base", +) -> Tuple[Optional[List], Optional[str]]: + """Fetch OHLCV data for any pool via GeckoTerminal + + Concurrent callers asking for the same upstream window share one request, no + matter what key each of them caches the answer under. + + Args: + pool_address: Pool contract address + network: Network identifier (will be converted to GeckoTerminal format) + timeframe: OHLCV timeframe (see GECKO_TIMEFRAMES) + currency: Price currency - "usd" or "token" (quote token) + user_data: Optional user_data dict for caching + limit: Number of candles to fetch (capped at GECKO_OHLCV_MAX) + before_timestamp: Unix seconds; candles walk back from here. None asks for + the latest candles, which is also what keeps a live chart's cache key + stable — pass it only for a window that has actually closed. + token: Which side of the pool to price, "base" or "quote". "quote" flips + the series, for a venue pair quoted the other way round from the pool + (``XRP-RLUSD`` against a ``RLUSD / XRP`` pool). + + Returns: + Tuple of (ohlcv_list, error_message) + ohlcv_list: List of [timestamp, open, high, low, close, volume] or None + error_message: Error string if failed, None on success + """ + try: + gecko_network = get_gecko_network(network) + timeframe = normalize_timeframe(timeframe) + limit = max(1, min(int(limit), GECKO_OHLCV_MAX)) + + # Check cache. limit/before_timestamp belong to the key: the same pool + # charted over a historical window and over the live one are different + # answers. + if user_data is not None: + cache_key = ( + f"ohlcv_{gecko_network}_{pool_address}_{timeframe}_{currency}" + f"_{token}_{limit}_{before_timestamp or 0}" + ) + cached = get_cached( + user_data, cache_key, ttl=OHLCV_CACHE_TTL, namespace=_CACHE_NS + ) + if cached is not None: + return cached, None + # Sweep on miss, as cached_call does. Historical windows mint a fresh + # key per executor, so a long-lived caller would otherwise accumulate + # one entry per chart forever. + evict_expired(user_data, namespace=_CACHE_NS) + + async def _fetch_upstream() -> Optional[List]: + # Pass all parameters explicitly: + # - currency="token" means price in quote token (not USD) + # - token="base" means OHLCV for the base token + result = await gecko_call( + "get_ohlcv", + gecko_network, + pool_address, + timeframe, + before_timestamp=before_timestamp, + currency=currency, + token=token, + limit=limit, + ) + + # Parse response - handle different formats + rows = None + + try: + import pandas as pd + + if isinstance(result, pd.DataFrame): + if not result.empty: + # Convert DataFrame to list format + rows = result.values.tolist() + except ImportError: + pass + + if rows is None: + if isinstance(result, list): + rows = result + elif isinstance(result, dict): + # Try nested structure + data = result.get("data", result) + if isinstance(data, dict): + attrs = data.get("attributes", data) + rows = attrs.get("ohlcv_list", []) + elif isinstance(data, list): + rows = data + + # Debug logging: show price range from OHLCV data + if rows: + try: + closes = [float(c[4]) for c in rows if len(c) > 4 and c[4]] + if closes: + logger.info( + f"OHLCV {pool_address[:8]}... {timeframe} currency={currency}: " + f"{len(rows)} candles, price range [{min(closes):.6f} - {max(closes):.6f}]" + ) + except Exception as e: + logger.debug(f"Could not log OHLCV price range: {e}") + + return rows + + # Coalesce on the UPSTREAM window — pool, timeframe, currency, priced side + # and window — never on the caller's cache key. The TTL caches above are + # per-caller and only help once an answer has landed, so two callers whose + # own keys differ (a chart keyed by trading pair and one keyed by pool, the + # dashboard's shared cache and a chat's user_data) still resolve to the very + # same GeckoTerminal window and would each spend a request for it. Keyed + # here, they collapse onto one. A fetch that raises reaches every waiter and + # is cached by nobody. The rows are shared between callers: read-only. + ohlcv_list = await _single_flight( + ( + "ohlcv", + gecko_network, + pool_address, + timeframe, + currency, + token, + limit, + before_timestamp or 0, + ), + _fetch_upstream, + ) + + if not ohlcv_list: + return None, "No OHLCV data available" + + # Cache result + if user_data is not None: + set_cached(user_data, cache_key, ohlcv_list, namespace=_CACHE_NS) + + return ohlcv_list, None + + except Exception as e: + logger.error(f"Error fetching OHLCV: {e}", exc_info=True) + return None, f"Failed to fetch OHLCV: {str(e)}" + + +# Bins for one pool, keyed ``(connector, pool_address)``. Used when no caller +# supplies a ``user_data`` to cache in — i.e. by the web dashboard, where every +# viewer of a pool would otherwise open its own Gateway call at the same cadence. +# Same one-minute TTL as the per-chat cache: bins move with every swap through +# the active bin, so a longer one would draw liquidity that has already left. +_pool_bins_cache: Dict[Tuple[str, str], Tuple[float, Dict[str, Any]]] = {} + + +async def fetch_liquidity_bins( + pool_address: str, + connector: str = "meteora", + network: str = "solana-mainnet-beta", + user_data: dict = None, + chat_id: int = None, + context=None, + client=None, +) -> Tuple[Optional[List], Optional[Dict], Optional[str]]: + """Fetch liquidity bin data for CLMM pools via gateway + + Args: + pool_address: Pool contract address + connector: DEX connector (meteora, raydium, orca) + network: Network identifier + user_data: Optional user_data dict for caching. When omitted the result + is cached process-wide instead, so callers with no conversation of + their own (the web dashboard) still share one Gateway call per pool. + chat_id: Chat ID for per-chat server selection + client: An already-resolved API client. The web side holds a server + *name* and resolves its client through the config manager, which the + ``get_client(chat_id)`` path below cannot express. + + Returns: + Tuple of (bins_list, pool_info, error_message) + bins_list: List of bin dicts with price, base_token_amount, quote_token_amount + pool_info: Full pool info dict + error_message: Error string if failed, None on success + """ + try: + if not can_fetch_liquidity(connector): + return None, None, f"Liquidity data not available for {connector}" + + # Check cache + cache_key = f"pool_bins_{connector}_{pool_address}" + if user_data is not None: + cached = get_cached( + user_data, cache_key, ttl=BINS_CACHE_TTL, namespace=_CACHE_NS + ) + else: + cached = _ttl_get( + _pool_bins_cache, (connector, pool_address), BINS_CACHE_TTL + ) + if cached is not None: + return cached.get("bins"), cached, None + + if client is None: + client = await get_client(chat_id, context=context) + if not client: + return None, None, "Gateway client not available" + + pool_info = None + + # First try get_pool_info (works for pools known to gateway) + try: + pool_info = await client.gateway_clmm.get_pool_info( + connector=connector, network=network, pool_address=pool_address + ) + except Exception as e: + # If get_pool_info fails (e.g., pool not in gateway config or not a DLMM pool), + # try finding the pool via get_pools search + error_str = str(e) + if "validation error" in error_str.lower() or "Field required" in error_str: + logger.info( + f"Pool {pool_address[:12]}... not found via get_pool_info, trying get_pools search" + ) + try: + # Search for pool by address using get_pools + search_result = await client.gateway_clmm.get_pools( + connector=connector, search_term=pool_address, limit=1 + ) + pools = search_result.get("pools", []) + if pools: + # Found the pool, but get_pools doesn't include bins + # Return pool info without bins - caller can handle this + pool_info = pools[0] + pool_info["address"] = pool_address + logger.info( + f"Found pool via get_pools: {pool_info.get('trading_pair', 'Unknown')}" + ) + else: + # Pool not found in DLMM pools - might be an AMM pool or non-existent + logger.info( + f"Pool {pool_address[:12]}... not found in {connector} DLMM pools" + ) + return ( + None, + None, + f"Pool not found in {connector} DLMM pools. This may be an AMM pool or not a {connector} pool.", + ) + except Exception as search_e: + logger.warning(f"get_pools search also failed: {search_e}") + return ( + None, + None, + f"Could not fetch pool info. Pool may not be a {connector} DLMM pool.", + ) + + if pool_info is None: + # Re-raise with a cleaner message for non-validation errors + return None, None, f"Failed to fetch pool: {str(e)[:100]}" + + if not pool_info: + return None, None, "Pool not found" + + bins = pool_info.get("bins", []) + + # Cache result + if user_data is not None: + set_cached(user_data, cache_key, pool_info, namespace=_CACHE_NS) + else: + _ttl_put( + _pool_bins_cache, (connector, pool_address), pool_info, BINS_CACHE_TTL + ) + + return bins, pool_info, None + + except Exception as e: + logger.error(f"Error fetching liquidity bins: {e}", exc_info=True) + return None, None, f"Failed to fetch liquidity: {str(e)}" + + +def _token_address_from_id(token_id: Any) -> str: + """``solana_`` → ````. Anything without the prefix passes through. + + GeckoTerminal ids are ``{network}_{address}`` in the relationships block, but + ``process_pools_list`` already strips them on its list endpoints and the + by-address endpoint does not. Neither a base58 pubkey nor an EVM 0x-address + contains an underscore, so stripping is idempotent and safe for both shapes. + """ + text = str(token_id or "").strip() + if not text or text.lower() == "nan": + return "" + _head, sep, tail = text.partition("_") + return tail if sep and tail else text + + +def _gecko_token_address(pool: dict, attrs: dict, side: str) -> str: + """The ``base``/``quote`` token address on a GeckoTerminal pool row. + + Reads the nested ``relationships.{side}_token.data.id`` the raw API returns, + then the flat ``{side}_token_id`` / ``{side}_token_address`` columns a + DataFrame row carries instead. Columns have moved between library versions, + so every known shape is tried before giving up. + """ + rel = ((pool.get("relationships") or {}).get(f"{side}_token") or {}).get("data") + nested = (rel or {}).get("id") if isinstance(rel, dict) else None + token_obj = attrs.get(f"{side}_token") + candidates = ( + nested, + attrs.get(f"{side}_token_id"), + attrs.get(f"{side}_token_address"), + token_obj.get("address") if isinstance(token_obj, dict) else None, + ) + for candidate in candidates: + address = _token_address_from_id(candidate) + if address: + return address + return "" + + +def normalize_pool_data(pool: dict, source: str = "gecko") -> Dict[str, Any]: + """Normalize pool data from different sources to a common format + + Args: + pool: Raw pool data dict + source: Data source ("gecko" or "gateway") + + Returns: + Normalized pool dict with consistent keys + """ + if source == "gecko": + # GeckoTerminal format + attrs = pool.get("attributes", pool) + + return { + "address": attrs.get("address") or pool.get("id", "").split("_")[-1], + "name": attrs.get("name", "Unknown"), + "base_token_symbol": attrs.get("base_token_symbol", "???"), + "quote_token_symbol": attrs.get("quote_token_symbol", "???"), + # A pool-first UI cannot do without the mints: an LP or DEX order + # executor's trading_pair is ``-``, so a pool + # with no base address is a pool nothing can be opened in. + "base_token_address": _gecko_token_address(pool, attrs, "base"), + "quote_token_address": _gecko_token_address(pool, attrs, "quote"), + "base_token_price_usd": attrs.get("base_token_price_usd"), + "quote_token_price_usd": attrs.get("quote_token_price_usd"), + "network": pool.get("network") or attrs.get("network", "solana"), + "dex_id": attrs.get("dex_id", "unknown"), + "reserve_usd": attrs.get("reserve_in_usd"), + "volume_24h": _get_nested_float(attrs, "volume_usd", "h24"), + "volume_6h": _get_nested_float(attrs, "volume_usd", "h6"), + "volume_1h": _get_nested_float(attrs, "volume_usd", "h1"), + "price_change_24h": _get_nested_float( + attrs, "price_change_percentage", "h24" + ), + "price_change_6h": _get_nested_float( + attrs, "price_change_percentage", "h6" + ), + "price_change_1h": _get_nested_float( + attrs, "price_change_percentage", "h1" + ), + "fdv_usd": attrs.get("fdv_usd"), + "market_cap_usd": attrs.get("market_cap_usd"), + "pool_created_at": attrs.get("pool_created_at"), + "source": "gecko", + } + + elif source == "gateway": + # Gateway CLMM format + return { + "address": pool.get("pool_address") or pool.get("address", ""), + "name": pool.get("trading_pair") or pool.get("name", "Unknown"), + "base_token_symbol": pool.get("base_symbol", "???"), + "quote_token_symbol": pool.get("quote_symbol", "???"), + "base_token_price_usd": None, # Not provided by gateway + "quote_token_price_usd": None, + "network": "solana", + "dex_id": pool.get("connector", "meteora"), + "reserve_usd": pool.get("liquidity") or pool.get("tvl"), + "volume_24h": pool.get("volume_24h"), + "price_change_24h": None, + "current_price": pool.get("current_price") or pool.get("price"), + "bin_step": pool.get("bin_step"), + "apr": pool.get("apr"), + "apy": pool.get("apy"), + "base_fee_percentage": pool.get("base_fee_percentage"), + "mint_x": pool.get("mint_x"), + "mint_y": pool.get("mint_y"), + "base_token_address": str(pool.get("mint_x") or ""), + "quote_token_address": str(pool.get("mint_y") or ""), + "source": "gateway", + } + + elif source == "orca": + return _normalize_orca_whirlpool(pool) + + return pool + + +# Orca states a fee tier in millionths of the trade: 400 is 0.04%. +_ORCA_FEE_RATE_DENOMINATOR = 10_000 + + +def _normalize_orca_whirlpool(pool: dict) -> Dict[str, Any]: + """One raw ``api.orca.so`` whirlpool → the same shape the other sources produce. + + Every figure Gateway's Orca listing leaves null is present here, so unlike a + Gateway row this one needs no GeckoTerminal backfill and no estimated yield: + ``apr`` is Orca's own realized fees (rewards included) over TVL, not volume + times the base tier. + """ + token_a = pool.get("tokenA") if isinstance(pool.get("tokenA"), dict) else {} + token_b = pool.get("tokenB") if isinstance(pool.get("tokenB"), dict) else {} + stats = pool.get("stats") if isinstance(pool.get("stats"), dict) else {} + day = stats.get("24h") if isinstance(stats.get("24h"), dict) else {} + + base_symbol = str(token_a.get("symbol") or "???") + quote_symbol = str(token_b.get("symbol") or "???") + mint_a = str(pool.get("tokenMintA") or token_a.get("address") or "") + mint_b = str(pool.get("tokenMintB") or token_b.get("address") or "") + + fee_rate = _finite(pool.get("feeRate")) + info: Dict[str, Any] = { + "address": str(pool.get("address") or ""), + "name": f"{base_symbol}-{quote_symbol}", + "base_token_symbol": base_symbol, + "quote_token_symbol": quote_symbol, + "base_token_address": mint_a, + "quote_token_address": mint_b, + # Carried under the Gateway row's names too, so a consumer that already + # reads a CLMM pool's mints does not need to know which source it came from. + "mint_x": mint_a, + "mint_y": mint_b, + "base_token_price_usd": None, + "quote_token_price_usd": None, + "network": "solana", + "dex_id": "orca", + "reserve_usd": pool.get("tvlUsdc"), + "current_price": pool.get("price"), + # Orca's tick spacing sits in the column Meteora's bin step does: both are + # "how finely can I place a position in this pool". + "bin_step": pool.get("tickSpacing"), + "base_fee_percentage": ( + fee_rate / _ORCA_FEE_RATE_DENOMINATOR if fee_rate is not None else None + ), + "volume_24h": day.get("volume"), + "fees_24h": day.get("fees"), + # Orca reports deltas as ratios (-0.05 = -5%); the table renders percents. + "price_change_24h": _scale_ratio(day.get("priceDelta")), + "source": "orca", + } + + # Fees *and* rewards over TVL, which is the number Orca's own UI shows — a + # reward-heavy pool's real yield is not its fee yield. + daily = _finite(day.get("yieldOverTvl")) + if daily is None: + daily = _finite(pool.get("yieldOverTvl")) + # Always present, even as None, so an Orca row and a Gateway one have the same + # keys and a consumer never has to ask which source it is holding. + info["apr"] = daily * 100 if daily is not None else None + info["apy"] = ( + ((1 + daily) ** _DAYS_PER_YEAR - 1) * 100 if daily is not None else None + ) + # Not estimated: no ``~`` on either column. + info["apr_estimated"] = False + + # Orca states every figure as a decimal *string*. The yields above are already + # floats, so coercing here — rather than leaving it to ``decorate_pool`` as the + # other sources do — is what keeps the row from being half one and half the + # other for anything that reads it before decoration. + return coerce_pool_numbers(info) + + +def _scale_ratio(value: Any) -> Optional[float]: + """A ratio from Orca as a percentage, or None when it reports nothing.""" + ratio = _finite(value) + return None if ratio is None else ratio * 100 + + +def _get_nested_float(data: dict, *keys) -> Optional[float]: + """Get a nested float value from dict, trying multiple key patterns""" + # Try nested access + value = data + for key in keys: + if isinstance(value, dict): + value = value.get(key) + else: + value = None + break + + if value is not None: + try: + return float(value) + except (ValueError, TypeError): + pass + + # Try flattened key with underscore + flat_key = "_".join(keys) + value = data.get(flat_key) + if value is not None: + try: + return float(value) + except (ValueError, TypeError): + pass + + # Try flattened key with dot + flat_key = ".".join(keys) + value = data.get(flat_key) + if value is not None: + try: + return float(value) + except (ValueError, TypeError): + pass + + return None + + +def extract_pair_from_name(name: str) -> Tuple[str, str]: + """Extract base and quote symbols from pool name + + Args: + name: Pool name like "SOL/USDC" or "SOL-USDC" or "SOL / USDC" + + Returns: + Tuple of (base_symbol, quote_symbol) + """ + if not name: + return "???", "???" + + # Try different separators + for sep in ["/", " / ", "-", " - "]: + if sep in name: + parts = name.split(sep) + if len(parts) >= 2: + return parts[0].strip(), parts[1].strip() + + return name, "???" + + +# ── Pool discovery ── +# The pool browser's three sources, behind one module. Telegram's /lp flow reaches +# GeckoTerminal and Gateway CLMM through its own inline fetching, each wrapped in +# MarkdownV2 table building and per-user caching, so none of it is callable from a +# web route. These are: the same upstreams, normalized, decorated and TTL-cached +# process-wide, with no presentation attached. + +GECKO_POOL_VIEWS = ("trending", "top", "new", "token") + +# GeckoTerminal serves every pool list in fixed pages of 20 and answers 401 past +# page 10, so those two numbers — not a preference of ours — bound what the pool +# browser can page through. +GECKO_PAGE_SIZE = 20 +GECKO_MAX_PAGE = 10 +# With a dex filter on, matching rows are scattered across upstream pages and have +# to be walked for. Capped because the walk costs one rate-limited request per +# page and the browser is asking for a screenful, not a census. +_GECKO_FILTER_WALK_PAGES = 3 +# Merging several venues' own listings costs one request per venue per page, so +# the fan-out is bounded: past this, the chain-wide walk is used instead. +_GECKO_SCOPED_MAX_REQUESTS = 8 + +# The list endpoint behind each view. The library exposes one method per view but +# none of them take ``page``, so the paths are used directly. +_GECKO_VIEW_PATHS = { + "trending": GECKO_CONSTANTS.GET_TRENDING_POOLS_BY_NETWORK_PATH, + "top": GECKO_CONSTANTS.GET_TOP_POOLS_BY_NETWORK_PATH, + "new": GECKO_CONSTANTS.GET_NEW_POOLS_BY_NETWORK_PATH, +} + +# A chain's venues change on the order of weeks; the filter dropdown reads this. +GECKO_DEXES_TTL = 6 * 3600 + +_gecko_page_cache: Dict[Tuple, Tuple[float, List[Dict[str, Any]]]] = {} +_gecko_dexes_cache: Dict[Tuple, Tuple[float, List[Dict[str, str]]]] = {} +_gateway_pool_list_cache: Dict[Tuple, Tuple[float, List[Dict[str, Any]]]] = {} +_orca_pool_list_cache: Dict[Tuple, Tuple[float, List[Dict[str, Any]]]] = {} +_multi_pool_cache: Dict[Tuple[str, str], Tuple[float, List[Dict[str, Any]]]] = {} +_pool_by_address_cache: Dict[Tuple[str, str], Tuple[float, Dict[str, Any]]] = {} + +# A pool list is a browser page, not a report: cap what an upstream can be asked +# for so one query cannot drag a thousand rows through normalization. +_POOL_LIST_MAX = 100 + + +def _coerce_pool_rows(result: Any, limit: int = 20) -> List[Dict[str, Any]]: + """``DataFrame | {"data": [...]} | [...]`` → a list of raw pool dicts. + + ``geckoterminal_py`` answers the pool-list endpoints with a pandas DataFrame, + but the raw JSON shapes turn up too (a hand-rolled ``api_request``, a recorded + fixture), and which one arrives has changed across library versions. Anything + that is not a mapping is dropped rather than crashing the whole listing. + """ + if result is None: + return [] + + try: + import pandas as pd + + if isinstance(result, pd.DataFrame): + rows = result.to_dict("records") + return [r for r in rows if isinstance(r, dict)][:limit] + except ImportError: # pragma: no cover - pandas ships with geckoterminal_py + pass + + if isinstance(result, dict): + rows = result.get("data") or [] + elif isinstance(result, list): + rows = result + else: + rows = getattr(result, "data", None) or [] + + if isinstance(rows, dict): # a by-address response: one pool, not a list + rows = [rows] + return [r for r in rows if isinstance(r, dict)][:limit] + + +# Every numeric a pool row carries, and the two upstreams disagree on the type of +# nearly all of them: GeckoTerminal answers strings (``"12500000.0"``) *and* +# pandas ``NaN`` for a missing figure, Gateway answers strings for APR and TVL. +# A NaN is not JSON (FastAPI refuses to encode one, and the whole listing 500s), +# and a string has no ``.toFixed``, so the browser crashes formatting it. Both are +# settled here, once, rather than in every consumer. +_POOL_NUMERIC_FIELDS = ( + "reserve_usd", + "volume_24h", + "volume_6h", + "volume_1h", + "price_change_24h", + "price_change_6h", + "price_change_1h", + "fdv_usd", + "market_cap_usd", + "current_price", + "base_token_price_usd", + "quote_token_price_usd", + "apr", + "apy", + "fees_24h", + "base_fee_percentage", +) + + +def _finite(value: Any) -> Optional[float]: + """``value`` as a float, or None when it is missing, unparseable or NaN.""" + if value is None or isinstance(value, bool): + return None + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if math.isfinite(number) else None + + +def coerce_pool_numbers(info: Dict[str, Any]) -> Dict[str, Any]: + """Make every numeric on a pool row a finite float (or None), in place.""" + for field in _POOL_NUMERIC_FIELDS: + if field in info: + info[field] = _finite(info[field]) + if "bin_step" in info: + step = _finite(info.get("bin_step")) + info["bin_step"] = int(step) if step is not None else None + return info + + +def decorate_pool(info: Dict[str, Any], network: str) -> Dict[str, Any]: + """Add the venue facts a pool-first UI cannot derive on its own, in place. + + A browser row has to answer three questions before the user clicks it — can I + chart this pool, can I trade in it, can I LP in it — and every one of them is + a fact about *our* connectors, not about the pool. Deciding them here keeps + the frontend from re-deriving venue rules it cannot know, and keeps "chart yes, + LP no" a renderable state rather than an executor that fails on create. + """ + dex_id = str(info.get("dex_id") or "") + gecko_network = get_gecko_network(network) + gateway_network = GECKO_TO_GATEWAY_NETWORK.get(gecko_network) or ( + network if network in NETWORK_TO_GECKO else "" + ) + + # normalize_pool_data reports "???" for a symbol GeckoTerminal only publishes + # inside the pool's display name; _fill_pool_pair_fields has parsed it by now. + def _symbol(side: str) -> str: + for key in (f"{side}_symbol", f"{side}_token_symbol"): + value = str(info.get(key) or "").strip() + if value and value != "???": + return value.upper() + return "" + + base_symbol, quote_symbol = _symbol("base"), _symbol("quote") + info["base_symbol"] = base_symbol or "???" + info["quote_symbol"] = quote_symbol or "???" + info["base_token_symbol"] = base_symbol or "???" + info["quote_token_symbol"] = quote_symbol or "???" + + provider = lp_provider_for_dex(dex_id, network) + info["lp_provider"] = provider + info["lp_supported"] = provider is not None + info["gateway_network"] = gateway_network + info["network"] = network + info["gecko_network"] = gecko_network + # A chain GeckoTerminal indexes is not necessarily one Gateway reaches. + info["tradable"] = bool(gateway_network) and gateway_network in NETWORK_TO_GECKO + # Consumed by the liquidity-bin depth chart (FEAT-045); decided here because + # it is the same kind of venue fact as lp_supported. + info["has_bins"] = can_fetch_liquidity(dex_id, network) + + # The - form LP and DEX order executors already carry, + # except on a network whose venues quote tickers rather than addresses (xrpl), + # where the base *is* a symbol. + base = base_symbol if uses_symbol_pairs(network) else info.get("base_token_address") + base = str(base or "") + info["trading_pair"] = f"{base}-{quote_symbol}" if base and quote_symbol else "" + return coerce_pool_numbers(info) + + +def _normalize_gecko_pool(row: Dict[str, Any], network: str) -> Dict[str, Any]: + """One raw GeckoTerminal pool row → a decorated, normalized pool.""" + info = normalize_pool_data(row, source="gecko") + _fill_pool_pair_fields(info, network, row) + return decorate_pool(info, network) + + +def _clamp_pool_limit(limit: Any) -> int: + try: + value = int(limit) + except (TypeError, ValueError): + return 20 + return max(1, min(value, _POOL_LIST_MAX)) + + +async def _gecko_page_rows( + gnet: str, view: str, token: str, page: int, dex: Optional[str] = None +) -> Optional[List[Dict[str, Any]]]: + """One raw upstream page of pool rows, or None when the fetch failed. + + Raw rather than normalized, because normalization depends on the *caller's* + network id (``solana-mainnet-beta`` and ``solana`` are one gecko chain but two + Gateway networks) while this cache is keyed by the gecko chain alone. + + None and ``[]`` mean different things and are cached differently: an empty page + really is the end of the list, a failure is not, and caching a rate-limit reply + for a minute would turn one throttled request into a minute of empty browsers. + """ + key = (gnet, view, token, page, dex or "") + cached = _ttl_get(_gecko_page_cache, key, POOL_LIST_TTL) + if cached is not None: + return [dict(row) for row in cached] + + # Concurrent askers for this exact page share one request. Rows are copied per + # caller below, since the shared answer is one list object. + rows = await _single_flight( + ("page",) + key, lambda: _gecko_page_fetch(gnet, view, token, page, dex, key) + ) + return None if rows is None else [dict(row) for row in rows] + + +async def _gecko_page_fetch( + gnet: str, + view: str, + token: str, + page: int, + dex: Optional[str], + key: Tuple, +) -> Optional[List[Dict[str, Any]]]: + """The uncached fetch behind ``_gecko_page_rows``. One upstream request.""" + if dex: + path = GECKO_CONSTANTS.GET_TOP_POOLS_BY_NETWORK_DEX_PATH.format(gnet, dex) + elif view == "token": + path = GECKO_CONSTANTS.GET_TOP_POOLS_BY_NETWORK_TOKEN_PATH.format(gnet, token) + else: + path = _GECKO_VIEW_PATHS[view].format(gnet) + + try: + payload = await gecko_request("GET", path, params={"page": page}) + rows = [ + row + for row in glom(payload, GECKO_CONSTANTS.POOL_SPEC) + if isinstance(row, dict) + ] + except Exception as e: + # Every rate-limit reply, and the 401 gecko answers past page 10. + # A stale copy beats an empty table: this listing is a ranked snapshot, so + # rows from a few minutes ago are still the answer to "what should I look + # at", while "No pools found" reads as a fact about the chain. The stale + # entry is returned but *not* refreshed, so the next request past the TTL + # tries upstream again rather than settling into serving old rows forever. + stale, age = _stale_get(_gecko_page_cache, key) + logger.warning( + "gecko pool page failed net=%s view=%s dex=%s page=%s: %s%s", + gnet, + view, + dex or "-", + page, + e, + f" (serving {age:.0f}s-old cache)" if stale is not None else "", + ) + if stale is not None: + return stale + return None + + _stale_put(_gecko_page_cache, key, rows) + return rows + + +def _normalized_gecko_rows( + rows: List[Dict[str, Any]], network: str, dexes: set +) -> List[Dict[str, Any]]: + """Raw upstream rows → decorated pools, keeping only the venues asked for.""" + pools: List[Dict[str, Any]] = [] + for row in rows: + try: + pool = _normalize_gecko_pool(row, network) + except Exception as e: + logger.info("gecko pool row skipped net=%s: %s", network, e) + continue + if not pool.get("address"): + continue + if dexes and str(pool.get("dex_id") or "").strip().lower() not in dexes: + continue + pools.append(pool) + return pools + + +async def _scoped_top_page( + gnet: str, network: str, venues: List[str], limit: int, page: int +) -> Dict[str, Any]: + """Top pools across several venues, from each venue's own listing. + + Filtering the chain-wide list instead would answer almost nothing: Solana's + top hundred pools are overwhelmingly one venue's, so "Top, on Meteora and + Orca" scanned that way finds a handful of rows and calls it the answer. Each + venue's own listing is already the top *of that venue*, so they are merged + and re-ranked by volume — which is what the chain-wide list ranks by too. + + Bounded by ``_GECKO_SCOPED_MAX_REQUESTS`` because the fan-out is one request + per venue per page, and every one of them shares the chart loop's budget. + """ + skip = (page - 1) * limit + pages_each = min(-(-(skip + limit) // GECKO_PAGE_SIZE), GECKO_MAX_PAGE) + budget = _GECKO_SCOPED_MAX_REQUESTS + + rows: List[Dict[str, Any]] = [] + for venue in venues: + for upstream in range(1, pages_each + 1): + if budget <= 0: + break + budget -= 1 + got = await _gecko_page_rows(gnet, "top", "", upstream, dex=venue) + if got is None: + break # this venue is as far as it goes; the others still count + rows.extend(got) + if len(got) < GECKO_PAGE_SIZE: + break + + pools = _normalized_gecko_rows(rows, network, set()) + pools.sort(key=lambda p: p.get("volume_24h") or 0.0, reverse=True) + return { + "pools": pools[skip : skip + limit], + "has_more": len(pools) > skip + limit, + } + + +async def list_gecko_pools_page( + network: str, + view: str = "trending", + token: Optional[str] = None, + limit: int = 20, + page: int = 1, + dexes: Optional[List[str]] = None, +) -> Dict[str, Any]: + """One page of GeckoTerminal pools, optionally narrowed to given venues. + + ``page`` is the browser's page, which is not the upstream's: gecko pages are a + fixed 20 rows, the browser picks its own size, and with a dex filter on the + matching rows are scattered across upstream pages. So the window + ``[(page-1)*limit, page*limit)`` is cut out of a walk over upstream pages — + started at the page that can hold it when unfiltered, and at the first page + when filtering, since only then do we know how many rows matched before it. + + ``has_more`` is answered from what the walk saw, not guessed from a full page: + a Next that lands on nothing is worse than no Next at all. + """ + from condor.dex_candles import ADDRESS_RE + + gnet = get_gecko_network(network) + view = (view or "trending").strip().lower() + if view not in GECKO_POOL_VIEWS: + view = "trending" + token = (token or "").strip() + if view == "token" and not ADDRESS_RE.match(token): + return {"pools": [], "has_more": False} + limit = _clamp_pool_limit(limit) + try: + page = max(1, int(page)) + except (TypeError, ValueError): + page = 1 + wanted = {d.strip().lower() for d in (dexes or []) if str(d).strip()} + + # One venue's *top* pools is a listing GeckoTerminal serves directly, already + # scoped and already paged — so that case costs one request instead of a walk, + # and pages as deep as the venue goes rather than as deep as we cared to scan. + # Trending and New have no venue-scoped form, so those still walk. + if view == "top" and len(wanted) > 1: + return await _scoped_top_page(gnet, network, sorted(wanted), limit, page) + scope = next(iter(wanted)) if view == "top" and len(wanted) == 1 else None + if scope: + wanted = set() + + skip = (page - 1) * limit + # With nothing to filter out, row N is on a known upstream page, so the walk + # starts there rather than re-fetching everything before it. + upstream = 1 if wanted else skip // GECKO_PAGE_SIZE + 1 + local_skip = skip if wanted else skip % GECKO_PAGE_SIZE + last_page = ( + min(GECKO_MAX_PAGE, upstream + _GECKO_FILTER_WALK_PAGES - 1) + if wanted + else GECKO_MAX_PAGE + ) + + collected: List[Dict[str, Any]] = [] + has_more = False + while upstream <= last_page: + rows = await _gecko_page_rows(gnet, view, token, upstream, dex=scope) + if rows is None: + # An upstream failure is not an empty chain: say nothing rather than + # claiming this view has no pools, and never cache the claim. Mid-walk + # though, the rows already in hand are a screenful the user can use — + # a rate limit hit on page 3 must not blank pages 1 and 2. + if not collected: + return {"pools": [], "has_more": False} + break + collected.extend(_normalized_gecko_rows(rows, network, wanted)) + if len(collected) > local_skip + limit: + has_more = True + break + if len(rows) < GECKO_PAGE_SIZE: + break # upstream has no more rows behind this page + upstream += 1 + if len(collected) >= local_skip + limit: + has_more = upstream <= last_page + break + + pools = collected[local_skip : local_skip + limit] + + # Trending and New have no venue-scoped form upstream, so a dex filter on them + # is a scan of the chain-wide list — and a venue that simply has nothing + # trending right now then reads as a venue with no pools at all. It has pools; + # they are just not in *this* ranking. So ask the venue's own top listing + # instead of answering "none", which is the query the user meant by picking it. + if not pools and wanted: + scoped = await _scoped_top_page(gnet, network, sorted(wanted), limit, page) + if scoped["pools"]: + return scoped + + return {"pools": pools, "has_more": has_more and len(pools) == limit} + + +async def list_gecko_pools( + network: str, + view: str = "trending", + token: Optional[str] = None, + limit: int = 20, +) -> List[Dict[str, Any]]: + """The first page of :func:`list_gecko_pools_page`, for callers with no paging.""" + result = await list_gecko_pools_page( + network, view=view, token=token, limit=limit, page=1 + ) + return result["pools"] + + +async def list_gecko_dexes(network: str) -> List[Dict[str, str]]: + """The venues GeckoTerminal indexes on a chain, for the browser's dex filter. + + The filter offers what the chain actually has rather than a hardcoded list, so + a new venue shows up without a deploy. An upstream failure answers ``[]``: a + filter that cannot be populated is a filter the user simply does not see. + """ + gnet = get_gecko_network(network) + cached = _ttl_get(_gecko_dexes_cache, (gnet,), GECKO_DEXES_TTL) + if cached is not None: + return [dict(row) for row in cached] + + try: + payload = await gecko_request( + "GET", GECKO_CONSTANTS.GET_DEXES_BY_NETWORK_PATH.format(gnet) + ) + rows = glom(payload, GECKO_CONSTANTS.DEXES_BY_NETWORK_SPEC) + except Exception as e: + # A chain's venue list changes about never, so a stale copy is as good as a + # fresh one — and losing it makes the filter vanish from the browser. + stale, age = _stale_get(_gecko_dexes_cache, (gnet,)) + logger.warning( + "gecko dex list failed net=%s: %s%s", + gnet, + e, + f" (serving {age:.0f}s-old cache)" if stale is not None else "", + ) + return [dict(row) for row in stale] if stale is not None else [] + + dexes = [ + {"id": str(row["id"]), "name": str(row.get("name") or row["id"])} + for row in rows + if isinstance(row, dict) and row.get("id") + ] + _ttl_put(_gecko_dexes_cache, (gnet,), dexes, GECKO_DEXES_TTL) + return [dict(row) for row in dexes] + + +# GeckoTerminal's multi-pool endpoint takes at most 30 addresses in one path. +_GECKO_MULTI_MAX = 30 + + +async def fetch_pools_by_addresses( + network: str, addresses: List[str] +) -> List[Dict[str, Any]]: + """Several pools by address in one request, for the favourites view. + + Favourites are stored as addresses, not as copies of the rows: a saved pool is + a pointer, and its TVL, volume and price have to be as live as any other row in + the table. Addresses that resolve to nothing are simply absent from the answer + — a pool that has since been dropped from the index is not an error. + """ + from condor.dex_candles import ADDRESS_RE + + gnet = get_gecko_network(network) + seen: List[str] = [] + for address in addresses: + address = str(address or "").strip() + if address and ADDRESS_RE.match(address) and address not in seen: + seen.append(address) + if not seen: + return [] + + # The favourites view re-reads every starred pool on every visit, and this was + # the one listing with no cache at all — a tab switch back and forth spent the + # chain listing's rate-limit budget. Keyed by the batch, so it collides with + # itself the way the other pool caches do. + pools: List[Dict[str, Any]] = [] + for start in range(0, len(seen), _GECKO_MULTI_MAX): + batch = seen[start : start + _GECKO_MULTI_MAX] + batch_key = (gnet, ",".join(batch)) + cached = _ttl_get(_multi_pool_cache, batch_key, POOL_LIST_TTL) + if cached is not None: + pools.extend(dict(row) for row in cached) + continue + try: + result = await gecko_call("get_multiple_pools_by_network", gnet, batch) + except Exception as e: + stale, age = _stale_get(_multi_pool_cache, batch_key) + logger.warning( + "gecko multi-pool failed net=%s: %s%s", + gnet, + e, + f" (serving {age:.0f}s-old cache)" if stale is not None else "", + ) + if stale is not None: + pools.extend(dict(row) for row in stale) + continue + batch_pools: List[Dict[str, Any]] = [] + for row in _coerce_pool_rows(result, len(batch)): + try: + pool = _normalize_gecko_pool(row, network) + except Exception as e: + logger.info("gecko pool row skipped net=%s: %s", gnet, e) + continue + if pool.get("address"): + batch_pools.append(pool) + _stale_put(_multi_pool_cache, batch_key, batch_pools) + pools.extend(dict(row) for row in batch_pools) + + # In the order the caller saved them, not the order gecko answers in. + rank = {address: i for i, address in enumerate(seen)} + pools.sort(key=lambda p: rank.get(str(p.get("address")), len(rank))) + return pools + + +def gateway_connector_network(connector: str) -> str: + """The network a Gateway CLMM connector's pools live on.""" + return LIQUIDITY_SUPPORTED_DEXES.get((connector or "").strip().lower(), "solana") + + +# A day is 365 compoundings of a daily rate — the same annualization Meteora's own +# `apy` applies to its `apr`, so a derived Orca figure sits on the same scale as +# the Meteora one in the row above it. +_DAYS_PER_YEAR = 365 + + +def _estimate_pool_yield(pool: Dict[str, Any]) -> None: + """Derive 24h fee/TVL and its annualization from volume and the fee tier. + + For a CLMM connector whose Gateway listing reports the fee tier and TVL but no + yield, the two yield columns would otherwise be empty for every one of its + pools. Fees are mechanical — volume times the tier — so with a volume from + GeckoTerminal the figure Meteora reports directly can be computed rather than + left blank. Orca no longer needs this: it is read from its own API, which + reports realized fees (see ``list_orca_pools``). + + Marked ``apr_estimated`` because it is: it assumes every trade paid the base + tier, which a dynamic-fee venue's real fees only approximate. + """ + volume = _finite(pool.get("volume_24h")) + tvl = _finite(pool.get("reserve_usd")) + fee_pct = _finite(pool.get("base_fee_percentage")) + if not volume or not tvl or fee_pct is None or tvl <= 0: + return + fees_24h = volume * fee_pct / 100 + daily = fees_24h / tvl + pool["fees_24h"] = fees_24h + pool["apr"] = daily * 100 + pool["apy"] = ((1 + daily) ** _DAYS_PER_YEAR - 1) * 100 + pool["apr_estimated"] = True + + +async def _fill_gateway_pool_gaps(pools: List[Dict[str, Any]], network: str) -> None: + """Fill the figures a CLMM connector leaves blank, from GeckoTerminal. + + Meteora's own listing answers volume, fees and APR, but no connector reports a + 24h price change, and a row with an em-dash under the columns that say whether + a pool is worth entering is barely a row. GeckoTerminal indexes the same pools + under the same addresses, so one multi-pool request per page closes the gap. + + Only rows with an actual gap are looked up, cached with the page they filled. + A failed lookup leaves the rows as they were: a blank column is what the + browser already renders. Orca used to be the worst case here — TVL and nulls — + and no longer passes through at all; it is read from its own API instead. + """ + wanted = [ + str(p.get("address")) + for p in pools + if p.get("address") + and (p.get("volume_24h") is None or p.get("price_change_24h") is None) + ] + if not wanted: + return + + try: + indexed = await fetch_pools_by_addresses(network, wanted) + except Exception as e: # pragma: no cover - fetch_pools_by_addresses swallows + logger.info("gateway pool enrichment failed net=%s: %s", network, e) + return + + by_address = {str(row.get("address")): row for row in indexed} + for pool in pools: + row = by_address.get(str(pool.get("address"))) + if not row: + continue + if pool.get("volume_24h") is None: + pool["volume_24h"] = row.get("volume_24h") + if pool.get("price_change_24h") is None: + pool["price_change_24h"] = row.get("price_change_24h") + # Gateway's TVL is the connector's own; gecko's only stands in for a gap. + if pool.get("reserve_usd") is None: + pool["reserve_usd"] = row.get("reserve_usd") + if pool.get("apr") is None and pool.get("apy") is None: + _estimate_pool_yield(pool) + + +async def list_gateway_pools( + client: Any, + connector: str, + search: Optional[str] = None, + limit: int = 20, + page: int = 1, +) -> List[Dict[str, Any]]: + """Gateway CLMM pools for a connector, optionally filtered by free text. + + Unlike the gecko sources this one is server-scoped (it needs a Hummingbot API + client) and searches *within* a connector by name rather than by token address + — which is why the route keeps the two behind a ``source`` discriminator + instead of one parameter set with meaningless combinations. + + ``page`` is 1-based like every other page in the browser; Gateway counts from + zero, and that translation stays here rather than in the route. + """ + connector = (connector or "").strip().lower() + if not connector: + return [] + search = (search or "").strip() or None + limit = _clamp_pool_limit(limit) + try: + page = max(1, int(page)) + except (TypeError, ValueError): + page = 1 + + key = (connector, search or "", limit, page) + cached = _ttl_get(_gateway_pool_list_cache, key, GATEWAY_POOL_LIST_TTL) + if cached is not None: + return [dict(row) for row in cached] + + if client is None or not hasattr(client, "gateway_clmm"): + logger.warning("Gateway CLMM unavailable for connector=%s", connector) + return [] + + try: + result = await client.gateway_clmm.get_pools( + connector=connector, page=page - 1, limit=limit, search_term=search + ) + except Exception as e: + logger.warning("gateway pool list failed connector=%s: %s", connector, e) + return [] + + raw = result.get("pools") if isinstance(result, dict) else result + network = gateway_connector_network(connector) + pools: List[Dict[str, Any]] = [] + for row in _coerce_pool_rows(raw, limit): + row = dict(row) + row.setdefault("connector", connector) + try: + pool = decorate_pool(normalize_pool_data(row, source="gateway"), network) + except Exception as e: + logger.info("gateway pool row skipped connector=%s: %s", connector, e) + continue + if pool.get("address"): + pools.append(pool) + + # Before the cache, so the extra upstream request is spent once per page and + # not once per viewer of it. + await _fill_gateway_pool_gaps(pools, network) + + _ttl_put(_gateway_pool_list_cache, key, pools, GATEWAY_POOL_LIST_TTL) + return [dict(row) for row in pools] + + +# How many whirlpools one snapshot holds. Orca lists over 8,000, but ranked by TVL +# fewer than ~500 hold more than $10k and the rest are dust the browser would never +# page to — so this is the whole browsable set, not a first page of it. +ORCA_SNAPSHOT_SIZE = 500 + + +async def _orca_snapshot(search: Optional[str]) -> Optional[List[Dict[str, Any]]]: + """Every browsable Orca whirlpool, normalized and ranked by TVL. + + ``None`` — not ``[]`` — when the upstream failed and there is no stale copy: + the caller falls back to the Gateway listing on that, and an empty table is + the wrong thing to show for a venue with thousands of pools. + """ + key = (search or "",) + cached = _ttl_get(_orca_pool_list_cache, key, ORCA_POOL_LIST_TTL) + if cached is not None: + return cached + + # Flicking between tabs asks for the same snapshot while the first request is + # still in the air; without this each one opens its own. + async def _fetch() -> Optional[List[Dict[str, Any]]]: + try: + rows = await orca_api.fetch_whirlpools(ORCA_SNAPSHOT_SIZE, search=search) + except Exception as e: + stale, age = _stale_get(_orca_pool_list_cache, key) + logger.warning( + "orca pool list failed search=%r: %s%s", + search or "", + e, + f" (serving {age:.0f}s-old cache)" if stale is not None else "", + ) + return stale + + network = gateway_connector_network("orca") + pools: List[Dict[str, Any]] = [] + for row in rows: + try: + pool = decorate_pool(normalize_pool_data(row, source="orca"), network) + except Exception as e: + logger.info("orca pool row skipped: %s", e) + continue + if pool.get("address"): + pools.append(pool) + _stale_put(_orca_pool_list_cache, key, pools) + return pools + + return await _single_flight(("orca-pools", key), _fetch) + + +async def list_orca_pools( + search: Optional[str] = None, limit: int = 20, page: int = 1 +) -> Optional[Tuple[List[Dict[str, Any]], bool]]: + """One page of Orca whirlpools, with whether another follows. + + Read straight from Orca rather than through Gateway, which proxies this very + API and drops volume, fees, price and yield on the way — and which ignores its + own ``page`` argument for this connector, so every page of the tab was page one. + + Sliced out of a cached snapshot because Orca paginates by cursor and the browser + pages by number: one upstream request per minute serves every page, which is + fewer than the request-per-page-plus-GeckoTerminal it replaces. ``has_more`` is + then a fact rather than the "a full page implies another" guess Gateway forces. + + ``None`` when the upstream failed and nothing stale was cached. + """ + limit = _clamp_pool_limit(limit) + try: + page = max(1, int(page)) + except (TypeError, ValueError): + page = 1 + + pools = await _orca_snapshot((search or "").strip() or None) + if pools is None: + return None + + start = (page - 1) * limit + window = pools[start : start + limit] + return [dict(row) for row in window], len(pools) > start + limit + + +async def fetch_pool_by_address( + network: str, pool_address: str +) -> Optional[Dict[str, Any]]: + """One normalized pool, for a deep link or a reload of the pool workspace. + + The workspace has to render from a URL alone, with no state carried over from + the browser that linked to it, so the pool is fetched by address rather than + looked up in a list. A pool that genuinely does not exist is cached as such; + a failed lookup is not. + """ + from condor.dex_candles import ADDRESS_RE + + pool_address = (pool_address or "").strip() + if not ADDRESS_RE.match(pool_address): + return None + + gnet = get_gecko_network(network) + key = (gnet, pool_address) + cached = _ttl_get(_pool_by_address_cache, key, POOL_LIST_TTL) + if cached is not None: + return dict(cached) if cached else None + + try: + # Shared with any concurrent viewer of the same pool — opening pools in + # quick succession otherwise opens one request each. + result = await _single_flight( + ("pool", gnet, pool_address), + lambda: gecko_call("get_pool_by_network_address", gnet, pool_address), + ) + except Exception as e: + logger.info("pool lookup failed net=%s pool=%s: %s", gnet, pool_address, e) + return None + + rows = _coerce_pool_rows(result, 1) + if not rows: + _ttl_put(_pool_by_address_cache, key, {}, POOL_LIST_TTL) + return None + + try: + pool = _normalize_gecko_pool(rows[0], network) + except Exception as e: + logger.info("pool parse failed net=%s pool=%s: %s", gnet, pool_address, e) + return None + + if not pool.get("address"): + pool["address"] = pool_address + _ttl_put(_pool_by_address_cache, key, pool, POOL_LIST_TTL) + return dict(pool) diff --git a/condor/preferences.py b/condor/preferences.py index c9c2acbeb..73ac5b9c7 100644 --- a/condor/preferences.py +++ b/condor/preferences.py @@ -1220,7 +1220,8 @@ def remove_custom_provider(user_data: Dict, name: str) -> bool: def unique_provider_name(user_data: Dict, suggested: str) -> str: """Return ``suggested`` (sanitized), suffixed with -2, -3, ... if taken.""" existing = { - sanitize_provider_name(p.get("name", "")) for p in get_custom_providers(user_data) + sanitize_provider_name(p.get("name", "")) + for p in get_custom_providers(user_data) } base = sanitize_provider_name(suggested) if base not in existing: diff --git a/condor/rates.py b/condor/rates.py index ee43436d3..2e8660c30 100644 --- a/condor/rates.py +++ b/condor/rates.py @@ -17,7 +17,7 @@ def _normalize(token: str) -> str: def merge_price_pool( - tickers_by_connector: Dict[str, Dict[str, dict]] + tickers_by_connector: Dict[str, Dict[str, dict]], ) -> Dict[str, float]: """Merge per-connector tickers into one `{pair: price}` pool. diff --git a/condor/reports/store.py b/condor/reports/store.py index 94e952fc4..7d00e2956 100644 --- a/condor/reports/store.py +++ b/condor/reports/store.py @@ -6,10 +6,11 @@ import contextvars import json import os -import tempfile from contextlib import contextmanager from pathlib import Path +from condor.fsutil import atomic_write_json, atomic_write_text + # reports/ is a repository-root output directory, not this source package. CHARTS_DIR = Path(__file__).resolve().parents[2] / "reports" INDEX_FILE = CHARTS_DIR / "reports_index.json" @@ -112,45 +113,12 @@ def _read_index() -> list[dict]: def _write_index(entries: list[dict]) -> None: - charts_dir = _charts_dir() - charts_dir.mkdir(exist_ok=True) - tmp = tempfile.NamedTemporaryFile( - mode="w", encoding="utf-8", dir=str(charts_dir), suffix=".tmp", delete=False - ) - try: - json.dump(entries, tmp, indent=2, ensure_ascii=False) - tmp.close() - os.replace(tmp.name, str(_index_file())) - except Exception: - tmp.close() - try: - os.unlink(tmp.name) - except OSError: - pass - raise + atomic_write_json(_index_file(), entries, indent=2, ensure_ascii=False) def _write_report_html(path: Path, content: str) -> None: """Atomically create or replace a report HTML file.""" - tmp = tempfile.NamedTemporaryFile( - mode="w", - encoding="utf-8", - dir=str(path.parent), - prefix=f".{path.name}.", - suffix=".tmp", - delete=False, - ) - try: - tmp.write(content) - tmp.close() - os.replace(tmp.name, path) - except Exception: - tmp.close() - try: - os.unlink(tmp.name) - except OSError: - pass - raise + atomic_write_text(path, content) def list_reports( diff --git a/condor/routine_hooks.py b/condor/routine_hooks.py index 521e01be2..2e2f7165f 100644 --- a/condor/routine_hooks.py +++ b/condor/routine_hooks.py @@ -1,27 +1,57 @@ -"""Persistent per-routine post-execution hooks. +"""Persistent per-routine, per-owner post-execution hooks. After a routine finishes, its report can be delivered to extra destinations: - - Telegram — the raw interactive .html report as a document to arbitrary - chat ids / groups. + - Telegram — the raw interactive .html report as a document. -Config is a persistent preset per routine name, stored in -``data/routine_hooks.json``, keyed by the routine name as known to each engine -(global base name, or ``slug/name`` for agent routines). It applies to every -execution of that routine (manual, scheduled, or after restart). +Config is a persistent preset stored in ``data/routine_hooks.json``, keyed by +the routine name as known to each engine (global base name, or ``slug/name`` +for agent routines) and then by the id of the user who configured it. It +applies to every execution of that routine *by that owner* (manual, scheduled, +or after restart). ``trigger`` controls when to fire: "success" (default), "always", or "failure". + +Ownership (SEC-152). Hooks used to be global and unowned: one config per +routine name, writable by any approved user, fired on every run by anyone. That +made them a silent exfiltration channel — attach a hook to a routine the admin +schedules against a server you have no access to and its full report (portfolio +value, PnL, positions) lands in your Telegram on every run. So now: + + - a hook belongs to the user who saved it, and only that user reads or + overwrites it; + - a run fires only the hooks owned by the user who started the run + (``user_id`` in the RoutineStore instance meta); + - recipients are bounded by what the owner controls — their own Telegram + chat — both when saving and again at dispatch, so a hooks file written by + an older version (or by hand) cannot deliver anywhere either. + +Admins are the exception at both ends: they already reach every server and +every instance, and they are the ones who route reports into shared groups, so +they may target any chat id. + +Legacy entries written before ownership existed are kept verbatim under the +``_UNOWNED`` key: they are nobody's, so they no longer fire for everyone — +only an admin's runs trigger them, and an admin saving that routine's hooks +adopts them under their own id. + +Dead ``email`` block (READ-172). Hooks once had an email destination; the +sender was deleted long ago, so configs written back then carry an ``email`` +block nothing reads and every save quietly dropped. It is now dropped on +*load* instead — see :data:`_DEAD_FIELDS` — so the config the dashboard shows, +the config dispatch acts on, and the config a save writes are the same thing, +and an old file on disk still loads without a fuss. """ from __future__ import annotations import json import logging -import os import re -import tempfile from pathlib import Path from typing import Any +from condor.fsutil import atomic_write_json + logger = logging.getLogger(__name__) _HOOKS_FILE = Path("data") / "routine_hooks.json" @@ -29,37 +59,123 @@ # Valid trigger conditions. _TRIGGERS = ("success", "always", "failure") +# Owner key for hooks saved before ownership existed. They are nobody's: kept +# on disk, readable and adoptable by an admin, fired only for an admin's runs. +_UNOWNED = "" + +# Config keys written by older builds that no code reads anymore (READ-172). +# Dropped when a stored entry is loaded, not when it is saved. +_DEAD_FIELDS = ("email",) + +# Routines already logged about, so a stale file does not repeat itself on +# every read. +_dead_field_warned: set[str] = set() + + +class ForbiddenRecipient(Exception): + """Raised when a hook targets a chat its owner is not allowed to reach.""" + + +# ── Ownership ── + + +def _is_admin(user_id: int | None) -> bool: + if not user_id: + return False + try: + from config_manager import get_config_manager + + return bool(get_config_manager().is_admin(int(user_id))) + except Exception: # noqa: BLE001 + # Fail closed: an unresolvable role is not an admin. + logger.warning("Could not resolve role for user %s", user_id, exc_info=True) + return False + + +def _owner_key(user_id: int | None) -> str: + return str(user_id) if user_id else _UNOWNED + + +def _allowed_recipient(chat_id: str, user_id: int | None) -> bool: + """Whether ``user_id`` may have this routine's reports delivered to ``chat_id``. + + A plain user only reaches their own Telegram chat — the one Condor already + talks to them in — so a hook can never carry a report somewhere its owner + does not control. Admins may target any chat: they route reports into + shared groups and channels, which is what the pre-SEC-152 hooks did. + """ + if not user_id: + return False + return chat_id == str(user_id) or _is_admin(user_id) + # ── Persistence ── -def _read_all() -> dict[str, dict]: +def _strip_dead_fields(cfg: Any, routine_name: str) -> Any: + """Drop config keys no code reads anymore, so a stale file still loads. + + A user's ``routine_hooks.json`` is live state we do not get to rewrite out + from under them, so removing a destination from the code has to leave the + old shape loadable. Dropping it here — rather than letting the next + ``save_hooks`` silently discard it — keeps what is read, shown and written + identical, and the discard shows up in the log instead of nowhere. + """ + if not isinstance(cfg, dict): + return cfg + dead = [k for k in _DEAD_FIELDS if k in cfg] + if not dead: + return cfg + if routine_name not in _dead_field_warned: + _dead_field_warned.add(routine_name) + logger.info( + "Dropping unused hook field(s) %s from %s; nothing has sent to them " + "since the email destination was removed", + ", ".join(dead), + routine_name, + ) + return {k: v for k, v in cfg.items() if k not in _DEAD_FIELDS} + + +def _normalize(entry: Any, routine_name: str) -> dict[str, dict]: + """Coerce one stored routine entry into ``{owner_key: config}``. + + Pre-SEC-152 files hold a bare config per routine name. Those entries are + preserved under :data:`_UNOWNED` rather than dropped or reassigned to a + made-up owner — losing an admin's delivery config silently would be as bad + a surprise as leaving it firing for everyone. + + A bare config is told apart from an owner map by its known keys. ``email`` + is deliberately not one of them: those files always carried ``telegram`` + and ``trigger`` alongside it, so it never discriminated anything, and it is + stripped before the check anyway. + """ + if not isinstance(entry, dict): + return {} + if "telegram" in entry or "trigger" in entry: + return {_UNOWNED: _strip_dead_fields(entry, routine_name)} + return { + str(k): _strip_dead_fields(v, routine_name) + for k, v in entry.items() + if isinstance(v, dict) + } + + +def _read_all() -> dict[str, dict[str, dict]]: if not _HOOKS_FILE.exists(): return {} try: data = json.loads(_HOOKS_FILE.read_text()) - return data if isinstance(data, dict) else {} + if not isinstance(data, dict): + return {} + return {name: _normalize(entry, name) for name, entry in data.items()} except Exception: # noqa: BLE001 logger.warning("Failed to read %s; treating as empty", _HOOKS_FILE) return {} -def _write_all(data: dict[str, dict]) -> None: - _HOOKS_FILE.parent.mkdir(parents=True, exist_ok=True) - tmp = tempfile.NamedTemporaryFile( - mode="w", dir=str(_HOOKS_FILE.parent), suffix=".tmp", delete=False - ) - try: - json.dump(data, tmp, indent=2) - tmp.close() - os.replace(tmp.name, str(_HOOKS_FILE)) - except Exception: - tmp.close() - try: - os.unlink(tmp.name) - except OSError: - pass - raise +def _write_all(data: dict[str, dict[str, dict]]) -> None: + atomic_write_json(_HOOKS_FILE, data, indent=2) def _default_config() -> dict: @@ -69,30 +185,61 @@ def _default_config() -> dict: } -def load_hooks(routine_name: str) -> dict | None: - """Return the hook config for a routine, or None if none set.""" - return _read_all().get(routine_name) +def load_hooks(routine_name: str, user_id: int | None) -> dict | None: + """Return ``user_id``'s hook config for a routine, or None if none set. + + An admin with no config of their own inherits the legacy unowned entry, so + the pre-SEC-152 hooks stay visible in the dashboard and are adopted the + next time they are saved. + """ + owners = _read_all().get(routine_name) or {} + own = owners.get(_owner_key(user_id)) + if own is not None: + return own + if _is_admin(user_id): + return owners.get(_UNOWNED) + return None + + +def save_hooks(routine_name: str, cfg: dict, user_id: int | None) -> dict: + """Validate and persist ``user_id``'s hook config. Returns the stored config. + Raises :class:`ForbiddenRecipient` if a chat id is not one the owner may + deliver to. + """ + if not user_id: + raise ForbiddenRecipient("Hooks need an owner") -def save_hooks(routine_name: str, cfg: dict) -> dict: - """Validate and persist the hook config for a routine. Returns the stored config.""" clean = _default_config() tg = cfg.get("telegram") or {} chat_ids = [str(c).strip() for c in (tg.get("chat_ids") or []) if str(c).strip()] # Light validation: chat ids are integers (may be negative for groups). chat_ids = [c for c in chat_ids if c.lstrip("-").isdigit()] + forbidden = [c for c in chat_ids if not _allowed_recipient(c, user_id)] + if forbidden: + raise ForbiddenRecipient( + "You can only send routine reports to your own chat " + f"({user_id}); not allowed: {', '.join(forbidden)}" + ) clean["telegram"] = {"enabled": bool(tg.get("enabled")), "chat_ids": chat_ids} trigger = cfg.get("trigger") clean["trigger"] = trigger if trigger in _TRIGGERS else "success" data = _read_all() + owners = data.setdefault(routine_name, {}) + # An admin writing this routine's hooks adopts whatever the pre-ownership + # file left behind, so the legacy entry stops shadowing their own config. + if _is_admin(user_id): + owners.pop(_UNOWNED, None) # If nothing is enabled/configured, drop the entry to keep the file tidy. if not clean["telegram"]["enabled"] and not chat_ids: - data.pop(routine_name, None) + owners.pop(_owner_key(user_id), None) else: - data[routine_name] = clean + owners[_owner_key(user_id)] = clean + if not owners: + data.pop(routine_name, None) _write_all(data) return clean @@ -145,13 +292,19 @@ async def dispatch( *, failed: bool, bot: Any | None, + owner_id: int | None, ) -> None: - """Send the configured Telegram notification for a routine. + """Send the Telegram notification ``owner_id`` configured for a routine. + + ``owner_id`` is who started this run — ``user_id`` in the RoutineStore + instance meta, the chat id on the Telegram path. Only that user's hooks + fire, so a run never delivers its report to someone else's destinations + (SEC-152). A run with no owner fires nothing. Never raises — failures are logged so a broken notification can't break the routine run itself. """ - cfg = load_hooks(routine_name) + cfg = load_hooks(routine_name, owner_id) if not cfg: return @@ -160,7 +313,15 @@ async def dispatch( return tg_cfg = cfg.get("telegram") or {} - if not (tg_cfg.get("enabled") and tg_cfg.get("chat_ids") and bot is not None): + # Re-check recipients here, not only at save time: a hooks file written by + # a pre-SEC-152 build (or edited by hand) must not be able to deliver + # anywhere its owner could not have configured today. + chat_ids = [ + c + for c in (tg_cfg.get("chat_ids") or []) + if _allowed_recipient(str(c), owner_id) + ] + if not (tg_cfg.get("enabled") and chat_ids and bot is not None): return # Send the raw interactive report HTML (live Plotly charts), or a minimal @@ -188,7 +349,7 @@ async def dispatch( slug += ".html" doc_bytes = html.encode("utf-8") - for chat_id in tg_cfg["chat_ids"]: + for chat_id in chat_ids: try: import io diff --git a/condor/routine_store.py b/condor/routine_store.py index e3ff7124b..4bb04491f 100644 --- a/condor/routine_store.py +++ b/condor/routine_store.py @@ -17,6 +17,7 @@ import condor.reports as reports from condor import routine_hooks +from condor.telemetry import taps as telemetry_taps from routines.base import ( RoutineResult, discover_routines, @@ -145,6 +146,42 @@ def user_data(self) -> dict: return self._user_data +# How many finished-run results stay resident (CORR-142). Results must outlive +# their run — the dashboard renders the detail panel after the run ends and only +# fetches the chart PNG when the user opens that run, and an MCP agent that +# submitted with `run_async` comes back for it via `get_instance` an unbounded +# time later. So this is deliberately generous: it is a memory backstop, not a +# cache TTL. What it prevents is the unbounded case — every `execute()` mints a +# fresh instance_id, and a RoutineResult can hold a raw PNG in `chart_image`, so +# an agent running a chart routine on a tick would otherwise pin megabytes a day +# in a process designed to run for weeks. +_MAX_RESULTS = 200 + +# CORR-163: the same bound, applied to the instance records themselves. +# `_results` was the expensive half of the leak (a RoutineResult can pin a raw +# PNG) but the `_instances` entry that owns it leaked in exactly the same shape: +# every `execute()` mints a fresh id, and only the explicit `remove_instance` +# and `stop` paths ever dropped one, so a completed one-shot was never reaped. +# +# Retention, not deletion-on-completion: a finished one-shot is precisely what +# the dashboard's detail panel renders after the run ends and what an MCP agent +# reads back through `get_instance` after a `run_async`, so the record has to +# outlive its own run. Only *terminal* instances count against the cap — +# anything in flight, continuous or scheduled is exempt from the count and from +# eviction, so a scheduled routine can never be reaped out from under its own +# task no matter how many one-shots run alongside it. +# +# Deliberately the same number as `_MAX_RESULTS`: an instance older than the +# result cap has already lost its result and is a hollow record, and sharing the +# bound is what keeps the two dicts from drifting apart. +_MAX_TERMINAL_INSTANCES = _MAX_RESULTS + +# Statuses a run cannot come back from. `running` (one-shot in flight, or a +# Telegram-owned instance, which keeps that status for its whole life) and +# `scheduled` (between ticks of an interval schedule) are the live ones. +_TERMINAL_STATUSES = frozenset({"completed", "failed", "stopped"}) + + class RoutineStore: """Singleton store for routine instances and results.""" @@ -266,15 +303,72 @@ def get_instance(self, instance_id: str) -> dict | None: def add_instance(self, instance_id: str, metadata: dict) -> None: self._instances[instance_id] = metadata + self._prune_instances() def remove_instance(self, instance_id: str) -> None: self._instances.pop(instance_id, None) self._tasks.pop(instance_id, None) + # The result dies with its instance. Every read path resolves the + # instance first (the dashboard detail and chart endpoints via + # _authorized_instance, the MCP tool via get_instance), so a result left + # behind here is unreachable memory rather than a cache. + self._results.pop(instance_id, None) + + def _has_live_task(self, instance_id: str) -> bool: + """True while the instance's own task is still on the loop.""" + task = self._tasks.get(instance_id) + return task is not None and not task.done() + + def _prune_instances(self) -> None: + """Evict the oldest terminal instances past ``_MAX_TERMINAL_INSTANCES``. + + The backstop for the unbounded case: every ``execute()`` mints a fresh + instance id and a one-shot that completes normally is removed by nobody, + so without this the dict is monotonic for the life of the process. + + Only instances in a terminal status are candidates, and only once their + task is off the loop — a run still in flight, a continuous routine and a + scheduled one between ticks are all exempt from eviction *and* from the + count, so no amount of one-shot churn can reap a live instance or push a + live one over the edge. The just-finished run is excluded on its own + prune for the same reason (its task is still executing this call); it is + the youngest candidate anyway, so it is never the one to go. + + Oldest-first by last run, falling back to creation for an instance that + never ran. Eviction goes through ``remove_instance`` so a dropped + instance takes its result and its task entry with it, preserving + CORR-142's ``set(_results) <= set(_instances)``. + """ + terminal = [ + (meta.get("last_run_at") or meta.get("created_at") or 0.0, iid) + for iid, meta in self._instances.items() + if meta.get("status") in _TERMINAL_STATUSES and not self._has_live_task(iid) + ] + excess = len(terminal) - _MAX_TERMINAL_INSTANCES + if excess <= 0: + return + terminal.sort() + for _, iid in terminal[:excess]: + self.remove_instance(iid) # ── Results ── def store_result(self, instance_id: str, result: RoutineResult) -> None: + """Record a run's result, keeping ``_results`` bounded to _MAX_RESULTS. + + Popping before inserting refreshes recency (dicts keep insertion order), + so a continuous instance that stores a result every tick stays at the + young end and the entries evicted are the oldest untouched ones. + + Not gated on the instance existing: the Telegram path calls this from + ``_execute_routine`` *before* it registers the instance via + ``_sync_instance_to_store``, so a gate here would drop every + Telegram-triggered result on its first run. + """ + self._results.pop(instance_id, None) self._results[instance_id] = result + while len(self._results) > _MAX_RESULTS: + self._results.pop(next(iter(self._results))) def get_result(self, instance_id: str) -> RoutineResult | None: return self._results.get(instance_id) @@ -292,14 +386,24 @@ async def _fire_hooks( routine_name = meta.get("routine_name") if meta else None if not routine_name: return + # Built outside the try on purpose (CORR-175). Binding the arguments + # happens here, at coroutine creation; nothing inside ``dispatch`` runs + # until it is awaited. So a call that no longer matches the signature + # raises TypeError *here* and surfaces, while every failure the dispatch + # itself hits stays swallowed below. The boundary is when the error + # happens, not its type: a TypeError raised while delivering is a + # runtime failure and must not break the run either. + coro = routine_hooks.dispatch( + routine_name, + result, + report_id, + failed=failed, + bot=(self._bot or _http_bot), + # Only the hooks of whoever started this run fire (SEC-152). + owner_id=meta.get("user_id"), + ) try: - await routine_hooks.dispatch( - routine_name, - result, - report_id, - failed=failed, - bot=(self._bot or _http_bot), - ) + await coro except Exception as e: logger.error( f"Post-execution hooks failed for {routine_name}[{instance_id}]: {e}" @@ -412,6 +516,7 @@ async def _execute_and_record( failed_status: str | None = None, fire_hooks: bool = True, agent: str = "", + trigger: str = "other", ) -> None: """Run a routine once, store the result, update instance metadata, fire hooks. @@ -455,12 +560,16 @@ async def _execute_and_record( duration = time.time() - start report_id = reports.get_last_report_id() - self._results[instance_id] = result # Clipped once: the instance record and the conversation note are the # same summary, so they cannot tell the user different stories. summary = result.text[:500] + # Both writes are gated on the instance still being there. A cancel that + # lands mid-run is recorded here *after* stop() already removed the + # instance, and storing the result then would resurrect an entry no read + # path can reach — unreachable and, without the gate, immortal. if instance_id in self._instances: + self.store_result(instance_id, result) self._instances[instance_id].update( { "status": ( @@ -478,6 +587,23 @@ async def _execute_and_record( "error": error_msg, } ) + # The run just reached its final status: this is the moment a new + # terminal instance appears, so it is the moment to enforce the cap. + self._prune_instances() + + # Usage telemetry (FEAT-023): whether it ran, how it was triggered and + # how long it took. The routine's own name only when the file ships in + # this repo — a user's private routine is called `custom` — and never + # its config, its output or its error text. + telemetry_taps.routine_run( + routine=telemetry_taps.shipped_routine_name( + (routine.name or "").split("/")[-1] + ), + kind="continuous" if getattr(routine, "continuous", False) else "oneshot", + trigger=trigger, + duration_ms=int(duration * 1000), + ok=not failed, + ) await self._report_run(instance_id, summary, error_msg) @@ -547,6 +673,7 @@ async def _run_oneshot( server_name: str, user_id: int = 0, agent: str = "", + trigger: str = "manual", ) -> None: await self._execute_and_record( instance_id, @@ -557,6 +684,7 @@ async def _run_oneshot( status_after="completed", failed_status="failed", agent=agent, + trigger=trigger, ) async def start_continuous( @@ -618,6 +746,7 @@ async def _run_continuous( status_after="stopped", fire_hooks=False, agent=agent, + trigger="manual", ) async def schedule( @@ -670,6 +799,7 @@ async def _run_scheduled( server_name, user_id, status_after="scheduled", + trigger="schedule", ) # A cancel that lands mid-run is swallowed (and recorded) by # _execute_and_record; stop() removed the instance, so bail out @@ -688,6 +818,9 @@ def stop(self, instance_id: str) -> bool: if instance_id in self._instances: self._instances[instance_id]["status"] = "stopped" del self._instances[instance_id] + # Same reasoning as remove_instance: stop() drops the instance + # record entirely, so its result is already unreachable. + self._results.pop(instance_id, None) return True return False diff --git a/condor/runtime/binding.py b/condor/runtime/binding.py index 533a90abd..c4113e465 100644 --- a/condor/runtime/binding.py +++ b/condor/runtime/binding.py @@ -112,9 +112,13 @@ def resolve( # and a specialist launched without it would read and write the CHAT's store. # What the slug must NOT decide down there is identity or routine scope — # see ``Settings.specialist_slug``. + # One derivation for both channels: these are the very ids sessions.py puts + # in CONDOR_USER_ID/CONDOR_CHAT_ID, and argv beats env in the subprocess, so + # a local fallback here would silently override the env one (SEC-180). + effective_user_id, effective_chat_id = spec.effective_ids() mcp_servers = build_mcp_servers_for_session( - spec.user_id or 0, - spec.chat_id or spec.user_id or 0, + effective_user_id, + effective_chat_id, user_data, server_name=effective_server if agent.server_required else None, agent_slug=agent.slug, diff --git a/condor/runtime/client.py b/condor/runtime/client.py index ad41437fd..1d50bd52a 100644 --- a/condor/runtime/client.py +++ b/condor/runtime/client.py @@ -14,13 +14,16 @@ import logging import os +import time from typing import AsyncIterator, Literal +from condor.acp.pydantic_ai_client import model_prefix from condor.runtime import conversations -from condor.runtime.events import RuntimeEvent +from condor.runtime.events import EventType, RuntimeEvent from condor.runtime.keys import SessionKey from condor.runtime.models import PromptRequest, SessionInfo, SessionSpec from condor.runtime.timeouts import TIMEOUTS +from condor.telemetry import taps as telemetry_taps log = logging.getLogger(__name__) @@ -57,6 +60,11 @@ def _local(): return sessions +def bound_agent_context(bound, user_id: int, platform: str) -> str: + """Opening context for a chat bound to a specialist Agent.""" + return _local().bound_agent_context(bound, user_id, platform) + + async def create_session( spec: SessionSpec, *, @@ -134,6 +142,27 @@ def _deny_pending_confirmations(raw_key: str) -> None: log.info("Denied %d pending confirmation(s) on steering %s", denied, raw_key) +# Session surfaces are short codes; the telemetry taxonomy spells them out. +_SURFACES = {"tg": "telegram", "web": "web", "mcp": "mcp", "": "other"} + + +def _turn_kind(req: PromptRequest, key: SessionKey) -> str: + """Which kind of turn this was, from what the request already carries.""" + kind = (getattr(req, "user_kind", "") or "").lower() + if kind in ("consult", "delegate", "tick"): + return kind + return "chat" + + +def _model_of(agent_key: str) -> str: + """The model id without any custom-endpoint nickname. + + ``custom@venice:llama-3.3-70b`` reports ``llama-3.3-70b``: the nickname is + something the operator typed and may name their employer or their host. + """ + return agent_key.split(":", 1)[1] if ":" in agent_key else agent_key + + async def prompt( key: SessionKey, req: PromptRequest, @@ -197,12 +226,25 @@ async def prompt( # as a system note rather than as the user's words. user_kind=req.user_kind, ) + # Turn shape for telemetry (FEAT-023): counts and categories only. Nothing + # about what was asked, answered, or which file a tool touched. + started = time.monotonic() + tool_kinds: dict[str, int] = {} + outcome = "aborted" try: async for event in session.prompt_stream(req.text, lock_timeout=lock_timeout): runtime_event = RuntimeEvent.from_acp(event, session_key=raw_key) + if runtime_event.type is EventType.TOOL_CALL: + # The ACP `kind` ("read", "execute", …), never the `title`: a + # title is free text and routinely contains a file path. + kind = str(runtime_event.field("kind") or "other") + tool_kinds[kind] = tool_kinds.get(kind, 0) + 1 + elif runtime_event.type is EventType.DONE: + outcome = "done" recorder.observe(runtime_event) yield runtime_event except Exception as exc: # noqa: BLE001 - surfaced to the caller as an event + outcome = "error" log.exception("Prompt failed for session %s", raw_key) failure = RuntimeEvent.error(str(exc), session_key=raw_key) recorder.observe(failure) @@ -214,6 +256,19 @@ async def prompt( # abandoned async generator only ever gets GeneratorExit. Losing the # half-written reply is the bug this feature exists to fix. recorder.flush() + # Here for the same reason: this is the one funnel Telegram, the + # dashboard and MCP all cross, and it is the only place that sees an + # abandoned turn end. + telemetry_taps.agent_turn( + kind=_turn_kind(req, key), + provider=model_prefix(session.agent_key) or "acp", + model=_model_of(session.agent_key), + tool_calls=sum(tool_kinds.values()), + duration_ms=int((time.monotonic() - started) * 1000), + outcome=outcome, + surface=_SURFACES.get(getattr(key, "surface", ""), "other"), + tools=tool_kinds, + ) async def prompt_once(key: SessionKey, text: str) -> str: diff --git a/condor/runtime/confirmations.py b/condor/runtime/confirmations.py index c065a7185..d18821e6e 100644 --- a/condor/runtime/confirmations.py +++ b/condor/runtime/confirmations.py @@ -26,6 +26,7 @@ from typing import Any, Protocol from condor.runtime.timeouts import TIMEOUTS +from condor.telemetry import taps as telemetry_taps log = logging.getLogger(__name__) @@ -44,6 +45,12 @@ class ConfirmationStatus(str, Enum): TIMEOUT = "timeout" +def _tool_of(pending: "PendingConfirmation") -> str: + """The tool name out of a tool_call payload, for telemetry only.""" + call = pending.tool_call or {} + return str(call.get("name") or call.get("kind") or "unknown") + + @dataclass class PendingConfirmation: """One approval request, awaiting a human.""" @@ -184,6 +191,9 @@ async def resolve( ) pending.selected_option_id = option_id pending._event.set() + # Which tool was asked about and what was decided (FEAT-023). Never the + # summary, the arguments or who decided. + telemetry_taps.confirmation(_tool_of(pending), "allow" if approved else "deny") return True def get(self, confirmation_id: str) -> PendingConfirmation | None: @@ -236,6 +246,7 @@ def sweep(self) -> int: ): pending.status = ConfirmationStatus.TIMEOUT pending._event.set() # wake any waiter immediately + telemetry_taps.confirmation(_tool_of(pending), "timeout") elif pending.status is not ConfirmationStatus.PENDING: # Give the waiter a grace period to read the outcome first. if now - pending.expires_at > CLEANUP_INTERVAL: diff --git a/condor/runtime/conversations.py b/condor/runtime/conversations.py index a6621b7df..e59e2f830 100644 --- a/condor/runtime/conversations.py +++ b/condor/runtime/conversations.py @@ -8,8 +8,9 @@ a subprocess is. The conversation is a directory: condor/.runtime/conversations/{user_id}/{conv_id}/ - meta.json # atomic merge, via registry_file.write_status() - transcript.jsonl # append-only, one JSON object per line + meta.json # atomic merge, via registry_file.write_status() + transcript.jsonl # append-only, bounded; one JSON object per line + transcript_archive.jsonl # turns retired out of the file above Same idiom as the rest of the runtime's durable facts: ``registry_file`` for atomic status, ``journal.py`` for an append-only narrative. No database. @@ -32,11 +33,14 @@ import shutil import time import uuid +from collections.abc import Iterator from datetime import datetime, timezone +from itertools import chain, islice from pathlib import Path from pydantic import BaseModel, ConfigDict, Field +from condor.fsutil import atomic_write_bytes from condor.runtime.events import EventType from condor.runtime.registry_file import read_status, write_status @@ -44,6 +48,7 @@ META_FILENAME = "meta.json" TRANSCRIPT_FILENAME = "transcript.jsonl" +TRANSCRIPT_ARCHIVE_FILENAME = "transcript_archive.jsonl" # conv_id and user_id both become directory names, so neither may escape one. # Same guard, same reason, as state.py's namespace check. @@ -59,14 +64,67 @@ REPLAY_HEADER = "Previously in this chat:" REPLAY_OMITTED = "(older turns omitted)" +# Bytes pulled per backwards read of a transcript. One block already covers a +# replay's whole char budget many times over; it only ever grows when a single +# turn is bigger than this. +_TAIL_BLOCK_BYTES = 64 * 1024 + +# Retention for the live transcript. The file is append-only, so without a +# sweep its footprint grows for the life of the conversation — PERF-138 made +# *reading* it cheap, not writing it. Past ``MAX`` bytes the oldest turns are +# **moved**, never deleted, to ``transcript_archive.jsonl`` beside it; the live +# file keeps the newest ``KEEP`` bytes plus one marker turn naming the sidecar +# and how many turns went into it. Same shape as PERF-136's journal archive and +# for the same reason: this is the user's own chat history, so a retention +# policy whose failure mode is "the chat is gone" is not one worth having. +# +# Bounded in bytes rather than in turns because bytes are what grow: one turn +# runs from ~100 B to ~3 KB (a tool call keeps up to ``TOOL_INPUT_MAX_CHARS`` + +# ``TOOL_OUTPUT_MAX_CHARS`` of payload), so a turn count would pin the +# footprint only to within a factor of thirty. It also makes the trigger a +# single ``stat()`` on the append path rather than a line count. +# +# MAX and KEEP are deliberately far apart: a trim leaves the file at KEEP, so +# the next one is a whole MAX−KEEP of appends away and the rewrite amortizes to +# a constant per byte appended. They are read from the environment here, beside +# ``REPLAY_MAX_CHARS`` and for the same reason, rather than from +# ``timeouts.py``, whose loader only understands deadlines in seconds. +TRANSCRIPT_MAX_BYTES = int( + os.environ.get("CONDOR_TRANSCRIPT_MAX_BYTES", "2097152") or 2097152 +) +TRANSCRIPT_KEEP_BYTES = int( + os.environ.get("CONDOR_TRANSCRIPT_KEEP_BYTES", "1048576") or 1048576 +) + +# The kept tail is never allowed below this multiple of the replay budget, and +# that is the load-bearing half of the policy: ``replay_context`` reads the live +# file only, so a trim must never be able to change what a resumed session is +# handed. Holding the tail well above ``REPLAY_MAX_CHARS`` guarantees a trim +# only ever removes turns that were already past the replay bound — the replay +# is identical before and after. A rendered replay line is a strict subset of +# the JSON line it came from, so bytes here dominate chars there; the multiple +# is slack on top of that. +_REPLAY_HEADROOM = 4 + +# The turn left behind in place of the archived ones. A real ``system`` entry +# rather than a bespoke line format, so every existing reader already handles +# it: the dashboard renders it like any other system note, ``read_transcript`` +# returns it, and ``_render_turn`` would replay it as "(N older turns moved to +# …)" if a replay window ever reached back that far. The count is folded +# forward on each trim, so the file states the running total and never claims +# to be a complete record when it is not. +ARCHIVE_MARKER_KIND = "archived" +_ARCHIVE_MARKER_RE = re.compile(r"^(\d+) older turns? moved to ") + TITLE_MAX_CHARS = 80 SNIPPET_MAX_CHARS = 160 # Upper bounds on the tool IO kept per call, in characters. Load-bearing, not # defensive: a tool result is routinely a market-data dump, a whole portfolio -# payload or a base64 image, and the transcript has no retention sweep — kept -# verbatim, one turn could outweigh the entire conversation and slow every -# read of the file. They live here, beside ``REPLAY_MAX_CHARS`` and for the +# payload or a base64 image — kept verbatim, one turn could outweigh the +# entire conversation, slow every read of the file and, since PERF-170, push +# the rest of the chat into the archive on its own. They live here, beside +# ``REPLAY_MAX_CHARS`` and ``TRANSCRIPT_MAX_BYTES``, and for the # same reason, rather than in ``timeouts.py``: that policy is # deadlines-in-seconds parsed from ``CONDOR_TIMEOUT_*``, and a character count # would not survive its loader. @@ -343,13 +401,57 @@ def list_conversations(user_id: int, *, limit: int = 100) -> list[ConversationMe return metas[:limit] if limit else metas -def read_transcript(user_id: int, conv_id: str, *, limit: int = 200) -> list[TurnEntry]: - """The tail of a transcript. ``limit=0`` means all of it. +def _iter_lines_reverse(path: Path, *, block: int | None = None) -> Iterator[bytes]: + """The file's non-blank lines, newest first, reading only what is consumed. - A line that fails to parse is skipped, never fatal — the same tolerance - ``read_status()`` shows for a truncated status file. + Opened in binary and split on ``b"\\n"``: no UTF-8 continuation byte is + ``0x0A``, so a block boundary can cut a character in half but never a line. + A record is always one line — ``append_turn`` writes ``json.dumps`` output, + which escapes newlines — so a line is a whole turn, and the last one is + yielded whether or not it ends in a newline (a torn append reads as a + malformed line, exactly as it does on the forward path). """ - path = _conv_dir(user_id, conv_id) / TRANSCRIPT_FILENAME + size = block or _TAIL_BLOCK_BYTES + with path.open("rb") as fh: + fh.seek(0, os.SEEK_END) + pos = fh.tell() + carry = b"" + while pos > 0: + step = min(size, pos) + pos -= step + fh.seek(pos) + lines = (fh.read(step) + carry).split(b"\n") + # The first piece may continue into the block before this one. + carry = lines.pop(0) + for line in reversed(lines): + line = line.strip() + if line: + yield line + carry = carry.strip() + if carry: + yield carry + + +def _iter_turns_reverse(path: Path) -> Iterator[TurnEntry]: + """Parsed turns, newest first, parsing only the lines the caller pulls. + + Same tolerance as the forward read: a line that will not parse is skipped, + an unreadable file is empty rather than fatal. + """ + try: + if not path.is_file(): + return + for line in _iter_lines_reverse(path): + try: + yield TurnEntry(**json.loads(line.decode("utf-8"))) + except Exception: # noqa: BLE001 - one bad line is not the file + continue + except OSError: + log.debug("Unreadable transcript at %s", path, exc_info=True) + + +def _read_turns(path: Path) -> list[TurnEntry]: + """Every turn in one file, oldest first. Absent or unreadable reads empty.""" entries: list[TurnEntry] = [] try: if not path.is_file(): @@ -366,7 +468,57 @@ def read_transcript(user_id: int, conv_id: str, *, limit: int = 200) -> list[Tur except OSError: log.debug("Unreadable transcript at %s", path, exc_info=True) return [] - return entries[-limit:] if limit else entries + return entries + + +def read_transcript( + user_id: int, + conv_id: str, + *, + limit: int = 200, + include_archive: bool = False, +) -> list[TurnEntry]: + """The tail of a transcript. ``limit=0`` means all of it. + + A line that fails to parse is skipped, never fatal — the same tolerance + ``read_status()`` shows for a truncated status file. + + A bounded tail is read backwards from the end, so a conversation with + thousands of turns costs its tail rather than its whole history; only + ``limit=0`` still walks the file. + + "All of it" means the live transcript, which retention bounds — a long + conversation's older turns live in the sidecar and are represented in the + result by the archive marker turn, so a caller is never quietly handed a + partial history that looks whole. ``include_archive=True`` is the full + record: the sidecar is read too and the marker is dropped, because with + both files in hand nothing is missing for it to mark. It costs the whole + history by definition, so it belongs to an export or a support dig, never + to a request path. + """ + conv_dir = _conv_dir(user_id, conv_id) + path = conv_dir / TRANSCRIPT_FILENAME + archive = conv_dir / TRANSCRIPT_ARCHIVE_FILENAME + + if limit: + turns: Iterator[TurnEntry] = _iter_turns_reverse(path) + if include_archive: + turns = ( + turn + for turn in chain(turns, _iter_turns_reverse(archive)) + if not _is_archive_marker(turn) + ) + tail = list(islice(turns, limit)) + tail.reverse() + return tail + + if not include_archive: + return _read_turns(path) + return [ + turn + for turn in chain(_read_turns(archive), _read_turns(path)) + if not _is_archive_marker(turn) + ] def append_turn(user_id: int, conv_id: str, entry: TurnEntry) -> None: @@ -385,6 +537,11 @@ def append_turn(user_id: int, conv_id: str, entry: TurnEntry) -> None: log.warning("Could not append turn to %s", conv_dir, exc_info=True) return + try: + _trim_transcript(conv_dir) + except Exception: # noqa: BLE001 - retention must not break a live prompt + log.warning("Could not trim transcript for %s", conv_dir, exc_info=True) + meta = get_conversation(user_id, conv_id) if meta is None: # The turn is on disk either way; without a meta there is nothing @@ -400,6 +557,115 @@ def append_turn(user_id: int, conv_id: str, entry: TurnEntry) -> None: write_status(conv_dir, META_FILENAME, **fields) +# ── Retention ── + + +def _keep_bytes() -> int: + """How much of the newest transcript a trim has to leave behind.""" + return max(TRANSCRIPT_KEEP_BYTES, REPLAY_MAX_CHARS * _REPLAY_HEADROOM) + + +def _is_archive_marker(turn: TurnEntry) -> bool: + return turn.role == "system" and turn.kind == ARCHIVE_MARKER_KIND + + +def _archived_count(line: bytes) -> int | None: + """Turns already archived according to ``line``; None if it is no marker. + + A marker we cannot read the count out of returns 0 rather than None: it is + still a marker, so it must be replaced instead of kept as a turn, and + under-counting the total beats emitting two markers. + """ + try: + data = json.loads(line.decode("utf-8")) + except (ValueError, UnicodeDecodeError): + return None + if not isinstance(data, dict) or data.get("kind") != ARCHIVE_MARKER_KIND: + return None + match = _ARCHIVE_MARKER_RE.match(str(data.get("text") or "")) + return int(match.group(1)) if match else 0 + + +def _archive_marker(count: int) -> bytes: + entry = TurnEntry( + role="system", + kind=ARCHIVE_MARKER_KIND, + text=f"{count} older turns moved to {TRANSCRIPT_ARCHIVE_FILENAME}", + ) + return json.dumps(entry.model_dump(mode="json"), ensure_ascii=False).encode("utf-8") + + +def _append_archive(conv_dir: Path, lines: list[bytes]) -> None: + """Append retired turns to the sidecar, durably. + + ``fsync``-ed before the caller republishes the live file, because until + this returns the live file is the only copy of these turns. + """ + with (conv_dir / TRANSCRIPT_ARCHIVE_FILENAME).open("ab") as fh: + fh.write(b"\n".join(lines) + b"\n") + fh.flush() + os.fsync(fh.fileno()) + + +def _trim_transcript(conv_dir: Path) -> None: + """Move the oldest turns of an oversized transcript into the sidecar. + + Archive first, then republish the live file, so every crash window fails + safe. Before the archive write: the transcript is untouched. Between the + two: the moved turns exist in *both* files, and a duplicate in a sidecar + nobody reads on the hot path is a far better outcome than a hole in + someone's chat. During the republish: ``atomic_write_bytes`` publishes by + rename, so a reader sees the whole old file or the whole new one and a + crash leaves the old one intact — the trim simply happens again on the + next append. There is no ordering in which a turn is lost. + + Whole lines only, and the newest turn is kept unconditionally: a single + turn larger than the entire budget has to shrink the file, not empty it. + """ + path = conv_dir / TRANSCRIPT_FILENAME + try: + if path.stat().st_size <= TRANSCRIPT_MAX_BYTES: + return + raw = path.read_bytes() + except OSError: + log.debug("Could not read transcript for trimming at %s", path, exc_info=True) + return + + lines = [line for line in raw.split(b"\n") if line.strip()] + if not lines: + return + archived = _archived_count(lines[0]) + if archived is not None: + lines.pop(0) + if not lines: + return + + budget = _keep_bytes() + cut = len(lines) - 1 + kept = len(lines[-1]) + 1 + for index in range(len(lines) - 2, -1, -1): + size = len(lines[index]) + 1 + if kept + size > budget: + break + kept += size + cut = index + + dropped, keep = lines[:cut], lines[cut:] + if not dropped: + return + + try: + _append_archive(conv_dir, dropped) + except OSError: + # Nowhere safe to park them, so nothing moves: the file stays over the + # bound and the next append retries the trim. + log.warning("Could not archive turns for %s", conv_dir, exc_info=True) + return + + marker = _archive_marker((archived or 0) + len(dropped)) + atomic_write_bytes(path, b"\n".join([marker, *keep]) + b"\n") + + def update_meta(user_id: int, conv_id: str, **fields) -> bool: """Merge fields into an existing conversation's meta. @@ -462,9 +728,23 @@ def replay_context( Walks backwards from the newest turn so the most recent ones always survive the bound, then reverses. The returned string never exceeds ``max_chars``. + + The walk reads the transcript backwards and parses lazily, so it stops at + the turn that overruns the budget instead of parsing the whole file first: + this runs on the session-spawn path. + + Only the live transcript is read, never the archive, and retention is sized + so that is not a compromise: ``_keep_bytes()`` holds the tail at several + times ``REPLAY_MAX_CHARS``, so a trim can only ever remove turns this walk + had already stopped short of. The replay a resumed session receives is + therefore byte-identical before and after a trim. Where the replay does + stop is stated in the output itself, by the ``REPLAY_OMITTED`` footer — + which has always been the honest boundary here, since the budget cuts the + history long before retention does. """ - turns = read_transcript(user_id, conv_id, limit=0) - if not turns: + turns = _iter_turns_reverse(_conv_dir(user_id, conv_id) / TRANSCRIPT_FILENAME) + newest_turn = next(turns, None) + if newest_turn is None: return "" # Budget the body so header and footer fit inside the caller's bound. @@ -476,10 +756,13 @@ def replay_context( lines: list[str] = [] used = 0 omitted = False - for turn in reversed(turns): + newest = "" + for turn in chain([newest_turn], turns): rendered = _render_turn(turn) if not rendered: continue + if not newest: + newest = rendered if used + len(rendered) + 1 > budget: omitted = True break @@ -489,7 +772,6 @@ def replay_context( if not lines: # Even the newest turn alone overruns the budget. Keeping a truncated # head of it beats returning nothing at all. - newest = next((r for r in (_render_turn(t) for t in reversed(turns)) if r), "") if not newest: return "" lines = [newest[:budget]] diff --git a/condor/runtime/loops.py b/condor/runtime/loops.py index 7ed36a9ed..6716f6faf 100644 --- a/condor/runtime/loops.py +++ b/condor/runtime/loops.py @@ -136,7 +136,15 @@ def record(self, engine, state: str) -> None: ) def record_tick(self, engine) -> None: - """Cheap per-tick counter update; keeps the last tick honest on a crash.""" + """Cheap per-tick counter update; keeps the last tick honest on a crash. + + Never for an engine that already left the registry: stop/shutdown wrote + its final state on the way out, and rewriting RUNNING over it would make + the next boot read a finished run as interrupted — and restart it when + the session opted into ``restart_on_boot``. + """ + if self._engines.get(engine.agent_id) is not engine: + return self.record(engine, LoopState.RUNNING) # ── Lifecycle ── diff --git a/condor/runtime/models.py b/condor/runtime/models.py index 0295bf1fa..efa4717d5 100644 --- a/condor/runtime/models.py +++ b/condor/runtime/models.py @@ -52,6 +52,23 @@ class SessionSpec(BaseModel): "a new one; a supplied id resumes that transcript.", ) + def effective_ids(self) -> tuple[int, int]: + """``(user_id, chat_id)`` as the MCP subprocess must see them. + + Both ids reach the subprocess twice — in its environment + (``CONDOR_USER_ID``/``CONDOR_CHAT_ID``) and on its argv + (``--user-id``/``--chat-id``, via ``binding.resolve``) — and argv wins + (mcp_servers/condor/settings.py). So the fallback for an ownerless or + chatless spec is derived here, once, and both channels read it: two + hand-copied ``or 0`` expressions used to disagree, and the subprocess + ran as the phantom user 0 that argv carried (SEC-180). + + A spec with neither id is legitimately ownerless — sessions.py supports + that shape deliberately — and lands on 0, which fails closed everywhere. + """ + chat_id = self.chat_id if self.chat_id is not None else (self.user_id or 0) + return self.user_id or chat_id, chat_id + class SessionInfo(BaseModel): """Serializable view of a live session.""" diff --git a/condor/runtime/registry_file.py b/condor/runtime/registry_file.py index 2822a00cf..01d8fb90f 100644 --- a/condor/runtime/registry_file.py +++ b/condor/runtime/registry_file.py @@ -20,6 +20,8 @@ from pathlib import Path from typing import Any +from condor.fsutil import atomic_write_json + log = logging.getLogger(__name__) STATUS_FILENAME = "status.json" @@ -71,17 +73,13 @@ def write_status( current["pid"] = os.getpid() current["updated_at"] = time.time() - tmp = path.with_suffix(".json.tmp") try: - path.parent.mkdir(parents=True, exist_ok=True) - tmp.write_text(json.dumps(current, indent=2), encoding="utf-8") - os.replace(tmp, path) # atomic within a filesystem + # Unique temp file per writer, inside the target's directory: several + # processes update the same status/meta file (loops, state, delegate), + # and a shared temp name would let them tear each other's write. + atomic_write_json(path, current, indent=2) except Exception: log.warning("Could not write status for %s", session_dir, exc_info=True) - try: - tmp.unlink(missing_ok=True) - except Exception: - pass def read_status( diff --git a/condor/runtime/sessions.py b/condor/runtime/sessions.py index c306f2833..d0a92d926 100644 --- a/condor/runtime/sessions.py +++ b/condor/runtime/sessions.py @@ -28,7 +28,11 @@ from condor.runtime.keys import SessionKey from condor.runtime.models import SessionInfo, SessionSpec from condor.runtime.timeouts import TIMEOUTS -from handlers.agents._shared import build_initial_context, get_project_dir +from handlers.agents._shared import ( + build_initial_context, + get_project_dir, + platform_formatting, +) log = logging.getLogger(__name__) @@ -155,6 +159,21 @@ async def prompt_stream(self, text: str, *, lock_timeout: float | None = None): PROMPT_OVERALL_TIMEOUT, self.key, ) + # Cancel at the agent before walking away, exactly as + # abort() does. Breaking out only stops us *relaying*: the + # turn would keep generating and keep running tools against + # a permission callback nobody is watching, and the next + # prompt would overlap it at the subprocess. Bounded by + # TIMEOUTS.prompt_cancel with a local fallback, so this + # cannot hang the caller. + try: + await self.client.abort_prompt() + except Exception: # noqa: BLE001 - never mask the timeout + log.warning( + "Could not cancel timed-out prompt for session %s", + self.key, + exc_info=True, + ) yield PromptDone(stop_reason="timeout") break finally: @@ -246,6 +265,29 @@ async def _enforce_session_budget(user_id: int) -> None: await _destroy_session_internal(victim.key) +def bound_agent_context( + bound: binding.SessionBinding, user_id: int, platform: str +) -> str: + """The opening context of a chat bound to a specialist Agent. + + A specialist opens with its own identity and domain memory rather than the + chat's context, which is what makes it a different brain instead of a skin + — but it meant it was also the one brain that never passed through + :func:`build_initial_context`, and so never heard how the surface it is + speaking into renders a reply: no tables and no charts on the dashboard, no + length rules on Telegram. The formatting section is appended here, at the + branch that skips the other path, so both teach it exactly once. + """ + return "\n\n".join( + ( + binding.agent_identity_context( + bound.agent_slug, user_id, bound.instructions, bound.label + ), + platform_formatting(platform), + ) + ) + + def _resolve_conversation( spec: SessionSpec, key: SessionKey, @@ -348,13 +390,13 @@ async def get_or_create_session( await _enforce_session_budget(spec.user_id) # MCP subprocess env expects a numeric chat id; surfaces without a chat - # (web, mcp) fall back to the user id. - effective_chat_id = ( - spec.chat_id if spec.chat_id is not None else (spec.user_id or 0) - ) + # (web, mcp) fall back to the user id. The same pair goes down on argv via + # binding.resolve, which is what the subprocess actually reads first — so + # both channels take it from one derivation (SEC-180). + effective_user_id, effective_chat_id = spec.effective_ids() extra_env = { "CONDOR_CHAT_ID": str(effective_chat_id), - "CONDOR_USER_ID": str(spec.user_id or effective_chat_id), + "CONDOR_USER_ID": str(effective_user_id), # Which session the MCP subprocess belongs to. The *conversation* id does # not exist yet here (it is minted below, after the client is up), but the # key does and is stable for the subprocess's whole life — so tools that @@ -458,9 +500,7 @@ async def get_or_create_session( # the chat's — that is what makes it a different brain, not a skin. initial_context = "" if bound.is_agent and spec.user_id: - initial_context = binding.agent_identity_context( - bound.agent_slug, spec.user_id, bound.instructions, bound.label - ) + initial_context = bound_agent_context(bound, spec.user_id, spec.platform) elif spec.user_id: initial_context = build_initial_context( spec.user_id, @@ -470,16 +510,30 @@ async def get_or_create_session( platform=spec.platform, server_name=spec.server_name, ) - # Resolve the server name that was actually used for this session - resolved_server = spec.server_name - if not resolved_server and spec.user_id: - from config_manager import get_config_manager, get_effective_server + # Resolve the server name that was actually used for this session. The + # chat default is the ambient answer, but ``chat_defaults`` is a global + # map keyed by chat id and ``spec`` is an unvalidated request body on + # the web, so naming someone else's chat would otherwise resolve their + # server here (SEC-178). Every candidate is therefore held to existence + # *and* reach, subjected on the run's own principal — the same id the + # MCP toolset is built for, so the label and the credentials downstream + # can never disagree about who this session belongs to. + from config_manager import get_config_manager, get_effective_server + + cm = get_config_manager() + subject_id, _ = spec.effective_ids() + + def usable(name: str | None) -> bool: + return bool(name and cm.get_server(name)) and cm.has_server_access( + subject_id, name + ) + resolved_server = spec.server_name + if not usable(resolved_server): resolved_server = get_effective_server(spec.chat_id, user_data) - if not resolved_server: - cm = get_config_manager() - accessible = cm.get_accessible_servers(spec.user_id) - resolved_server = accessible[0] if accessible else None + if not usable(resolved_server): + accessible = cm.get_accessible_servers(subject_id) + resolved_server = accessible[0] if accessible else None # The durable conversation behind this session. An empty id mints a new # one; a supplied id replays that transcript's tail into the opening diff --git a/condor/runtime/timeouts.py b/condor/runtime/timeouts.py index 90c27ba96..5a7e40cbb 100644 --- a/condor/runtime/timeouts.py +++ b/condor/runtime/timeouts.py @@ -43,7 +43,8 @@ class TimeoutPolicy: sse_stream: int = 1800 # Budget for one MCP tool call. mcp_call: float = 15.0 - # Default budget for one strategy tick. + # Wall-clock budget for one agent session: a strategy tick's LLM turn, and + # the shutdown cleanup pass that runs under the same ceiling. 10 minutes. tick_default: int = 600 @classmethod @@ -78,15 +79,16 @@ def resolve_tick_timeout( caller: int | None = None, strategy: int | None = None, ) -> int: - """Pick a tick budget: explicit caller override > strategy config > default. + """Pick an agent-session budget: caller override > run config > policy. - A single-tick run (dry run / run once) is a human waiting on a result, so it - gets the shorter default; a loop tick is unattended and may take longer. + ``strategy`` is the run's ``tick_timeout_sec``; 0 or None means "no opinion", + which falls through to ``tick_default`` (10 min, or CONDOR_TIMEOUT_TICK_DEFAULT). + ``execution_mode`` is accepted so a future policy can price attended runs + (dry_run / run_once) differently from unattended loops; today they share one + budget, and saying so here beats a branch that pretends otherwise. """ if caller: return int(caller) if strategy: return int(strategy) - if execution_mode in ("dry_run", "run_once"): - return TIMEOUTS.tick_default return TIMEOUTS.tick_default diff --git a/condor/server_data_service.py b/condor/server_data_service.py index 1d3cea7c4..318f50fa8 100644 --- a/condor/server_data_service.py +++ b/condor/server_data_service.py @@ -22,6 +22,8 @@ from functools import partial from typing import Any, Callable, Dict, FrozenSet, List, Optional, Tuple +from condor.asyncutil import TaskSet + logger = logging.getLogger(__name__) @@ -34,6 +36,7 @@ class ServerDataType(Enum): """All data types fetchable from a Hummingbot server.""" PORTFOLIO = "portfolio" + PORTFOLIO_HISTORY = "portfolio_history" PRICES = "prices" POSITIONS = "positions" ACTIVE_ORDERS = "active_orders" @@ -45,50 +48,78 @@ class ServerDataType(Enum): CANDLE_CONNECTORS = "candle_connectors" SERVER_STATUS = "server_status" ALL_CONNECTORS = "all_connectors" + VENUES = "venues" TICKERS = "tickers" TICKER_POOL = "ticker_pool" @dataclass(frozen=True) class DataTypeDefaults: - """Default interval, TTL, and stale threshold for a data type.""" + """Default polling interval and cache TTL for a data type.""" interval: float # Default polling interval (seconds) - ttl: float # How long cached data is considered valid without subscribers - stale_threshold: float # Data younger than this is always returned + ttl: float # How long cached data is considered valid + # Optional per-param TTL override: (param_name, {param_value: ttl}). Some + # data types cache variants that go stale at very different rates — a 1D + # portfolio-history window is worthless after minutes while a 3M one holds + # for hours — and the key params say which variant this is. + ttl_by_param: Optional[Tuple[str, Dict[str, float]]] = None + + def ttl_for(self, params: Dict[str, str]) -> float: + """TTL for a specific cache key's params.""" + if not self.ttl_by_param: + return self.ttl + param_name, table = self.ttl_by_param + return table.get(params.get(param_name, ""), self.ttl) + + def interval_for(self, params: Dict[str, str]) -> float: + """Default poll cadence for a specific cache key's params. + + Derived from that key's own TTL rather than a second per-param table: + polling a variant far more often than its data goes stale is pure + waste, and two tables that must agree eventually stop agreeing. The + declared ``interval:ttl`` ratio is what carries over, so a variant that + stays valid 60x longer is polled 60x less often. + """ + if not self.ttl_by_param or not self.ttl: + return self.interval + return self.interval * (self.ttl_for(params) / self.ttl) _DEFAULTS: Dict[ServerDataType, DataTypeDefaults] = { - ServerDataType.PORTFOLIO: DataTypeDefaults(interval=10, ttl=60, stale_threshold=5), - ServerDataType.PRICES: DataTypeDefaults(interval=3, ttl=30, stale_threshold=2), - ServerDataType.POSITIONS: DataTypeDefaults(interval=10, ttl=60, stale_threshold=5), - ServerDataType.ACTIVE_ORDERS: DataTypeDefaults( - interval=10, ttl=60, stale_threshold=5 - ), - ServerDataType.TRADING_RULES: DataTypeDefaults( - interval=300, ttl=600, stale_threshold=30 - ), - ServerDataType.CONNECTORS: DataTypeDefaults( - interval=300, ttl=600, stale_threshold=30 - ), - ServerDataType.BOTS_STATUS: DataTypeDefaults(interval=5, ttl=30, stale_threshold=3), - ServerDataType.EXECUTORS: DataTypeDefaults(interval=2, ttl=30, stale_threshold=1), - ServerDataType.BOT_RUNS: DataTypeDefaults(interval=30, ttl=120, stale_threshold=10), - ServerDataType.CANDLE_CONNECTORS: DataTypeDefaults( - interval=300, ttl=600, stale_threshold=30 - ), - ServerDataType.SERVER_STATUS: DataTypeDefaults( - interval=60, ttl=120, stale_threshold=15 + ServerDataType.PORTFOLIO: DataTypeDefaults(interval=10, ttl=60), + # One entry per range window, each with its own freshness horizon: a 1D + # window at 5m candles is stale within minutes, a 3M window at 1d candles + # holds for hours. ``interval == ttl`` here, so ``interval_for`` polls each + # range exactly at its own TTL instead of refetching the 3M series 60x per + # window like the dashboard's old shared 120s loop did. + ServerDataType.PORTFOLIO_HISTORY: DataTypeDefaults( + interval=120, + ttl=120, + ttl_by_param=( + "range_key", + {"1D": 120, "1W": 600, "1M": 3600, "3M": 7200}, + ), ), - ServerDataType.ALL_CONNECTORS: DataTypeDefaults( - interval=300, ttl=600, stale_threshold=30 - ), - ServerDataType.TICKERS: DataTypeDefaults(interval=60, ttl=180, stale_threshold=30), + ServerDataType.PRICES: DataTypeDefaults(interval=3, ttl=30), + ServerDataType.POSITIONS: DataTypeDefaults(interval=10, ttl=60), + ServerDataType.ACTIVE_ORDERS: DataTypeDefaults(interval=10, ttl=60), + ServerDataType.TRADING_RULES: DataTypeDefaults(interval=300, ttl=600), + ServerDataType.CONNECTORS: DataTypeDefaults(interval=300, ttl=600), + ServerDataType.BOTS_STATUS: DataTypeDefaults(interval=5, ttl=30), + ServerDataType.EXECUTORS: DataTypeDefaults(interval=2, ttl=30), + ServerDataType.BOT_RUNS: DataTypeDefaults(interval=30, ttl=120), + ServerDataType.CANDLE_CONNECTORS: DataTypeDefaults(interval=300, ttl=600), + ServerDataType.SERVER_STATUS: DataTypeDefaults(interval=60, ttl=120), + ServerDataType.ALL_CONNECTORS: DataTypeDefaults(interval=300, ttl=600), + # Venue traits follow the CONNECTORS cadence, not the "chains change ~never" + # one: the credentialed connector list is one of the inputs, so the answer + # changes the moment a user adds API keys. + ServerDataType.VENUES: DataTypeDefaults(interval=300, ttl=600), + ServerDataType.TICKERS: DataTypeDefaults(interval=60, ttl=180), # Whole-server ticker pool: one poll feeds every per-connector ticker view and # all currency conversion, so reads never hit the network. - ServerDataType.TICKER_POOL: DataTypeDefaults( - interval=60, ttl=300, stale_threshold=60 - ), + ServerDataType.TICKER_POOL: DataTypeDefaults(interval=60, ttl=300), } @@ -265,6 +296,10 @@ def __init__(self): self._last_cleanup = time.time() # Sync listeners (e.g. WebSocketManager broadcasts) self._listeners: List[Callable] = [] + # Strong refs to fire-and-forget subscriber-callback tasks so the GC + # can't cancel them mid-flight (the event loop only keeps a weak + # reference). Entries auto-remove on completion. + self._callback_tasks = TaskSet(logger, "SDS callback error for %s: %s") # ------ Fetch registry ------ @@ -306,13 +341,15 @@ async def subscribe( ) -> CacheKey: """Subscribe to data updates. Returns the CacheKey. - If interval is 0 or not provided, the default for the data type is used. + If interval is 0 or not provided, the data type's default cadence is + used — derived per key from its own TTL, so a long-lived variant is + polled as rarely as its freshness contract allows. Callback signature: async callback(key: CacheKey, old_value, new_value) """ + key = CacheKey.make(server, data_type, **params) if interval <= 0: - interval = _DEFAULTS[data_type].interval + interval = _DEFAULTS[data_type].interval_for(key.params_dict) - key = CacheKey.make(server, data_type, **params) sub = Subscription( subscriber_id=subscriber_id, key=key, @@ -373,12 +410,8 @@ def get(self, server: str, data_type: ServerDataType, **params) -> Optional[Any] defaults = _DEFAULTS[data_type] age = time.time() - entry.fetched_at - # Always return if within stale threshold - if age <= defaults.stale_threshold: - return entry.value - - # Past the stale threshold: still usable until the TTL expires - return entry.value if age <= defaults.ttl else None + # Cached data is usable until the TTL expires + return entry.value if age <= defaults.ttl_for(key.params_dict) else None async def get_or_fetch( self, server: str, data_type: ServerDataType, **params @@ -448,40 +481,29 @@ def invalidate_server(self, server: str) -> None: len(keys_to_remove), ) - # ------ Listener compatibility (for WebSocketManager) ------ + # ------ Cache-write listeners (for WebSocketManager) ------ def add_listener(self, callback: Callable) -> None: - """Add a sync listener: callback(server_name, cache_key_str, data_type_name, value)""" + """Add a sync listener: ``callback(key: CacheKey, value: Any)``. + + Fired on every cache write. The listener gets the typed key — server, + data type and params — so it never has to parse anything back out of a + formatted string. + """ self._listeners.append(callback) def remove_listener(self, callback: Callable) -> None: - self._listeners = [cb for cb in self._listeners if cb is not callback] + # Compared by equality, not identity: the only listener is a bound + # method (``ws_manager._on_data_update``), and attribute access builds a + # fresh method object every time, so ``is`` never matched the one that + # was registered and stop() left a dead manager wired to the singleton. + self._listeners = [cb for cb in self._listeners if cb != callback] def _notify_listeners(self, key: CacheKey, value: Any) -> None: - """Notify listeners with server/channel/data_type/value arguments. - - The WS manager uses the data_type name string to map to channels. - """ - if not self._listeners: - return - - # Build a channel-compatible cache_key string - params = key.params_dict - if key.data_type == ServerDataType.PRICES: - cache_key_str = f"cex_prices:{params.get('connector_name', '')}:{params.get('trading_pair', '')}" - elif key.data_type == ServerDataType.PORTFOLIO: - account = params.get("account_name", "") - cache_key_str = f"cex_balances:{account}" if account else "portfolio" - elif key.data_type == ServerDataType.BOTS_STATUS: - cache_key_str = "bots_status" - elif key.data_type == ServerDataType.EXECUTORS: - cache_key_str = "executors" - else: - cache_key_str = key.data_type.value - + """Hand the written key and its value to every listener, unchanged.""" for cb in self._listeners: try: - cb(key.server, cache_key_str, key.data_type, value) + cb(key, value) except Exception as e: logger.debug("SDS listener error: %s", e) @@ -633,6 +655,7 @@ def stop(self) -> None: if self._poll_task and not self._poll_task.done(): self._poll_task.cancel() logger.info("ServerDataService stopped") + self._callback_tasks.cancel_all() # ------ Poll loop ------ @@ -768,16 +791,24 @@ async def _do_fetch_and_cache(self, key: CacheKey) -> Optional[Any]: return result def _fire_callbacks(self, key: CacheKey, old_value: Any, new_value: Any) -> None: - """Fire subscriber callbacks asynchronously.""" + """Fire subscriber callbacks asynchronously. + + Each callback runs as a tracked task: a failing subscriber is logged + and never blocks the others or the caller. + """ subs = self._subscriptions.get(key) if not subs: return for sub in subs.values(): if sub.callback: try: - asyncio.ensure_future(sub.callback(key, old_value, new_value)) + task = asyncio.ensure_future( + sub.callback(key, old_value, new_value) + ) except Exception as e: logger.debug("SDS callback error for %s: %s", sub.subscriber_id, e) + else: + self._callback_tasks.track(task, sub.subscriber_id) def _cleanup_stale(self) -> None: """Remove cache entries with no subscribers and expired TTL.""" @@ -787,7 +818,10 @@ def _cleanup_stale(self) -> None: if key in self._subscriptions and self._subscriptions[key]: continue # Active subscribers, keep defaults = _DEFAULTS.get(key.data_type) - if defaults and now - entry.fetched_at > defaults.ttl * 2: + if ( + defaults + and now - entry.fetched_at > defaults.ttl_for(key.params_dict) * 2 + ): stale.append(key) for key in stale: del self._cache[key] @@ -849,16 +883,19 @@ def register_default_fetches() -> None: fetch_current_price, fetch_executors, fetch_portfolio, + fetch_portfolio_history, fetch_positions, fetch_server_status, fetch_ticker_pool, fetch_tickers, fetch_trading_rules, + fetch_venues, ) sds = get_server_data_service() sds.register_fetch(ServerDataType.PORTFOLIO, fetch_portfolio) + sds.register_fetch(ServerDataType.PORTFOLIO_HISTORY, fetch_portfolio_history) sds.register_fetch(ServerDataType.PRICES, fetch_current_price) # strict=True on the cached reads: a failed call must reach # _do_fetch_and_cache's except branch (record_error, keep the last good @@ -874,6 +911,7 @@ def register_default_fetches() -> None: ServerDataType.CONNECTORS, partial(fetch_available_cex_connectors, strict=True) ) sds.register_fetch(ServerDataType.ALL_CONNECTORS, fetch_connectors) + sds.register_fetch(ServerDataType.VENUES, partial(fetch_venues, strict=True)) sds.register_fetch(ServerDataType.BOTS_STATUS, fetch_bots_status) sds.register_fetch(ServerDataType.EXECUTORS, fetch_executors) sds.register_fetch(ServerDataType.BOT_RUNS, fetch_bot_runs) diff --git a/condor/telemetry/__init__.py b/condor/telemetry/__init__.py new file mode 100644 index 000000000..577606190 --- /dev/null +++ b/condor/telemetry/__init__.py @@ -0,0 +1,82 @@ +"""Anonymous, opt-in usage telemetry (FEAT-023). + +Condor is self-hosted, so the project sees nothing about how installs are used +unless an install chooses to tell it. This package is that channel — and, +because the same process holds exchange API keys, it is built to be auditable in +one sitting rather than to be clever. + +The four facts that define it: + +- **Off by default.** A fresh clone has no consent recorded, which resolves to + level ``off``, at which :func:`emit` returns before doing anything at all. No + buffer, no file, no directory. +- **Opt-in, once, by the admin.** One inline-keyboard prompt on boot offers + full usage, install-count-only, or no. The answer is durable and reversible. +- **Allowlisted.** :mod:`condor.telemetry.schema` declares every event and every + property. Anything undeclared is dropped by construction, which is what makes + the "never collected" list in ``PRIVACY.md`` a property of the code. +- **Nothing transmits without an endpoint.** No collector URL is compiled in. + Unset ``CONDOR_TELEMETRY_URL`` means the send path is inert and events only + ever reach a capped local file. + +Public surface — call sites should need nothing else:: + + from condor import telemetry + telemetry.emit("command", name="portfolio", surface="telegram") +""" + +from condor.telemetry.consent import ( + DENIED, + GRANTED, + OFF, + PING, + UNKNOWN, + USAGE, + is_on, + level, +) +from condor.telemetry.consent import state as consent_state +from condor.telemetry.emitter import emit, flush + +__all__ = [ + "emit", + "flush", + "level", + "is_on", + "consent_state", + "init", + "shutdown", + "OFF", + "PING", + "USAGE", + "UNKNOWN", + "GRANTED", + "DENIED", +] + + +def init(hosted: bool = True) -> str: + """Prime this process. Returns the effective level. + + ``hosted`` marks a process that owns a flush job, so events buffer in memory + instead of going straight to a spool file. The MCP subprocess passes False. + + Priming does two things and only two: it resolves the level once so + :func:`emit` never has to read the disk on a hot path, and — only if the + level is not ``off`` — it materializes the install's random ids. An install + that never opted in is left exactly as it was found. + """ + from condor.telemetry import consent, emitter + + emitter.set_hosted(hosted) + effective = consent.refresh() + if effective != consent.OFF: + consent.ensure_identity() + return effective + + +def shutdown(reason: str = "signal") -> None: + """Record that the process is going down. Sending is still the job's problem.""" + from condor.telemetry import context + + emit("shutdown", reason=reason, uptime_h=context.uptime_h()) diff --git a/condor/telemetry/consent.py b/condor/telemetry/consent.py new file mode 100644 index 000000000..d59c84167 --- /dev/null +++ b/condor/telemetry/consent.py @@ -0,0 +1,255 @@ +"""Consent: the three-state machine that decides whether anything happens at all. + +The install — not the individual user — is the unit of consent, because the +admin owns the install. The states are ``unknown`` (the default on a fresh +clone), ``granted`` and ``denied``, stored in ``config.yml`` under ``telemetry`` +alongside the install's identity. + +Two rules matter more than the rest: + +**Unknown means off.** Not "buffered until someone decides" — off. An install +whose admin has never answered the prompt collects nothing, holds nothing in +memory, and writes no spool file. The original design buffered pre-consent +events and simply refused to send them; recording data before permission is a +worse default than losing it, so this narrows that decision. Nothing is lost +that we had any right to. + +**The environment wins.** ``CONDOR_TELEMETRY`` in ``utils/config.py`` overrides +the stored answer in both directions, so a headless or containerized install is +never blocked on a Telegram prompt, and an operator can force ``off`` no matter +what is on disk. + +Reads never create ``config.yml``. On an install that has not been configured at +all, :func:`state` answers ``unknown`` from nothing. +""" + +from __future__ import annotations + +import logging +import time +import uuid +from pathlib import Path + +log = logging.getLogger(__name__) + +OFF = "off" +PING = "ping" +USAGE = "usage" +LEVELS = (OFF, PING, USAGE) + +UNKNOWN = "unknown" +GRANTED = "granted" +DENIED = "denied" + +# Answer -> level, the three buttons of the admin prompt. +ANSWER_LEVELS = {"usage": USAGE, "ping": PING, "off": OFF} + +_cached_level: str | None = None + + +def _config_path() -> Path: + from config_manager import ConfigManager + + existing = ConfigManager._instance + return Path(existing.config_path if existing else "config.yml") + + +def _cm(): + """The ConfigManager, but only if reading it cannot create a config file.""" + from config_manager import ConfigManager + + if ConfigManager._instance is None and not Path("config.yml").exists(): + return None + try: + return ConfigManager.instance() + except Exception: # pragma: no cover - a broken config must not break a tap + log.debug("Telemetry could not read config", exc_info=True) + return None + + +def _section() -> dict: + cm = _cm() + if cm is None: + return {} + try: + return cm.get_telemetry() + except Exception: # pragma: no cover + return {} + + +def _update(**changes) -> None: + cm = _cm() + if cm is None: + from config_manager import get_config_manager + + cm = get_config_manager() + cm.update_telemetry(**changes) + refresh() + + +def _env_level() -> str | None: + """The operator's override, or None when unset/nonsense.""" + from utils.config import CONDOR_TELEMETRY + + if CONDOR_TELEMETRY in LEVELS: + return CONDOR_TELEMETRY + if CONDOR_TELEMETRY: + log.warning( + "Ignoring CONDOR_TELEMETRY=%r: expected one of %s", + CONDOR_TELEMETRY, + ", ".join(LEVELS), + ) + return None + + +def state() -> str: + """``unknown`` | ``granted`` | ``denied``.""" + value = _section().get("consent") + return value if value in (UNKNOWN, GRANTED, DENIED) else UNKNOWN + + +def level() -> str: + """The effective level. Cached, because :func:`emit` reads it every time.""" + global _cached_level + if _cached_level is None: + _cached_level = _compute_level() + return _cached_level + + +def _compute_level() -> str: + env = _env_level() + if env is not None: + return env + if state() != GRANTED: + return OFF + stored = _section().get("level") + return stored if stored in LEVELS else USAGE + + +def refresh() -> str: + """Drop the cached level and recompute it. Called after any state change.""" + global _cached_level + _cached_level = None + return level() + + +def is_on() -> bool: + return level() != OFF + + +def env_overridden() -> bool: + """True when the operator has pinned the level, so no prompt should be sent.""" + return _env_level() is not None + + +# ── Identity ───────────────────────────────────────────────────────────── +# Both ids are random. Neither is derived from a MAC address, a hostname, a +# username or a token, so neither can be reversed into anything about the host. + + +def ensure_identity() -> dict: + """Create the install's ids if they do not exist yet. Writes ``config.yml``. + + Called from :func:`condor.telemetry.init` only when telemetry is actually + on, so a fresh install that never opts in never grows the section — and + ``emit()`` never has to touch the disk to find an id. + """ + section = _section() + changes = {} + if not section.get("install_id"): + changes["install_id"] = uuid.uuid4().hex + if not section.get("install_secret"): + changes["install_secret"] = uuid.uuid4().hex + if changes: + _update(**changes) + section = _section() + return section + + +def install_id() -> str: + return _section().get("install_id") or "" + + +def install_secret() -> str: + """Never transmitted, never logged. Only ever salts a local hash.""" + return _section().get("install_secret") or "" + + +# ── Transitions ────────────────────────────────────────────────────────── + + +def grant(answer: str) -> str: + """Record a positive answer. ``answer`` is one of :data:`ANSWER_LEVELS`.""" + chosen = ANSWER_LEVELS.get(answer, USAGE) + if chosen == OFF: + return deny() + _update( + consent=GRANTED, + level=chosen, + decided_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + ) + ensure_identity() + return chosen + + +def deny() -> str: + """Record a refusal and destroy whatever was already on disk. + + Denial deletes the spool and the outbox rather than merely ignoring them: + an answer of "no" should leave nothing behind to be sent by a later bug. + """ + _update( + consent=DENIED, + level=OFF, + decided_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + ) + from condor.telemetry import emitter, outbox + + emitter.discard_buffer() + outbox.purge() + return OFF + + +def set_level(new_level: str) -> str: + """Change the level of an install that has already consented.""" + if new_level not in LEVELS: + return level() + if new_level == OFF: + return deny() + _update(consent=GRANTED, level=new_level) + ensure_identity() + return new_level + + +# ── First-use tracking ─────────────────────────────────────────────────── + + +def mark_feature_seen(feature: str) -> bool: + """True the first time this install ever uses ``feature``, False after. + + Backs the ``feature_first_use`` activation funnel. Only called when + telemetry is on, so an opted-out install never accumulates the list. + """ + section = _section() + seen = list(section.get("features_seen") or []) + if feature in seen: + return False + seen.append(feature) + _update(features_seen=seen[-200:]) + return True + + +# ── Prompt bookkeeping ─────────────────────────────────────────────────── + + +def should_prompt(version: str = "") -> bool: + """Has this install never been asked (or not since this version)?""" + if env_overridden() or state() != UNKNOWN: + return False + asked = _section().get("prompted_version") + return asked != (version or "unknown") + + +def mark_prompted(version: str = "") -> None: + """Written *before* the prompt is sent, so a crash loop cannot re-ask forever.""" + _update(prompted_version=version or "unknown") diff --git a/condor/telemetry/context.py b/condor/telemetry/context.py new file mode 100644 index 000000000..e5202e826 --- /dev/null +++ b/condor/telemetry/context.py @@ -0,0 +1,147 @@ +"""The install context: who is reporting, not who is using. + +Everything here describes the *deployment* — a random id, a git sha, a platform +triple, and a handful of counts and booleans. It is computed once per process +and sent once per batch rather than once per event, which is both cheaper and a +smaller surface to audit. + +What is deliberately not here: hostname, IP, MAC, username, home directory, +timezone, locale, server names or URLs, and any count that could be traced to a +person. ``user_count`` is a number, never a list. +""" + +from __future__ import annotations + +import logging +import os +import platform +import subprocess +import sys +import time +from pathlib import Path + +log = logging.getLogger(__name__) + +_REPO = Path(__file__).resolve().parent.parent.parent +_started_at = time.monotonic() + +_app: dict | None = None + + +def uptime_h() -> float: + return round((time.monotonic() - _started_at) / 3600.0, 3) + + +def _git(*args: str) -> str: + """A short, bounded git read. Returns '' rather than raising, ever.""" + try: + out = subprocess.run( + ["git", *args], + cwd=str(_REPO), + capture_output=True, + text=True, + timeout=3, + ) + return out.stdout.strip() if out.returncode == 0 else "" + except Exception: + return "" + + +def _in_docker() -> bool: + try: + if Path("/.dockerenv").exists(): + return True + cgroup = Path("/proc/self/cgroup") + return cgroup.exists() and "docker" in cgroup.read_text() + except Exception: + return False + + +def app() -> dict: + """Version and platform. Computed once — the git calls are not free.""" + global _app + if _app is None: + # An install deployed from a tarball rather than a clone reports + # "unknown", which the collector treats as its own bucket. + _app = { + "version": _git("rev-parse", "--short", "HEAD") or "unknown", + "branch": _git("rev-parse", "--abbrev-ref", "HEAD") or "unknown", + "python": f"{sys.version_info.major}.{sys.version_info.minor}", + "os": platform.system().lower(), + "arch": platform.machine().lower(), + "in_docker": _in_docker(), + } + return dict(_app) + + +def version() -> str: + return app()["version"] + + +def _llm_providers() -> list[str]: + """Which provider *slots* are configured. Names only — never a key, never a + prefix of a key, never its length.""" + found = [] + for name, var in ( + ("openai", "OPENAI_API_KEY"), + ("anthropic", "ANTHROPIC_API_KEY"), + ("openrouter", "OPENROUTER_API_KEY"), + ("google", "GEMINI_API_KEY"), + ("groq", "GROQ_API_KEY"), + ("deepseek", "DEEPSEEK_API_KEY"), + ): + if os.environ.get(var, "").strip(): + found.append(name) + return found + + +def config_shape() -> dict: + """Counts and capability flags. Every field is a bool, an int, or a fixed name.""" + shape: dict = { + "has_web": True, + "has_gateway": False, + "has_hb_api": False, + "llm_providers": _llm_providers(), + "user_count": 0, + "server_count": 0, + "agent_count": 0, + } + try: + from condor.telemetry.consent import _cm + + cm = _cm() + if cm is not None: + servers = cm.list_servers() or {} + shape["server_count"] = len(servers) + shape["has_hb_api"] = bool(servers) + shape["user_count"] = len(cm.get_all_users() or []) + except Exception: + log.debug("Telemetry could not read config shape", exc_info=True) + try: + agents_dir = _REPO / "agents" + shape["agent_count"] = sum( + 1 for p in agents_dir.iterdir() if (p / "AGENT.md").is_file() + ) + except Exception: + pass + try: + shape["has_gateway"] = bool(os.environ.get("GATEWAY_URL", "").strip()) + except Exception: + pass + return shape + + +def envelope(events: list[dict], dropped: int, level: str) -> dict: + """The wire format. One context, many events, one idempotency key each.""" + from condor.telemetry import consent + + return { + "schema": 1, + "install_id": consent.install_id(), + "level": level, + "sent_at": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "app": app(), + "config": config_shape(), + "dropped": dropped, + "events": events, + } diff --git a/condor/telemetry/emitter.py b/condor/telemetry/emitter.py new file mode 100644 index 000000000..b740b5468 --- /dev/null +++ b/condor/telemetry/emitter.py @@ -0,0 +1,178 @@ +"""The emitter: one function, four gates, and a contract that makes it safe. + +``emit()`` is called from order paths, agent turns and error handlers. Its +contract is what makes that acceptable: + +1. **It never raises.** Every failure inside it is swallowed. Telemetry that can + break the bot is worse than no telemetry, so the whole body is wrapped and + the fallback is silence. +2. **It never blocks and never does I/O** in the host process. It appends to a + bounded ``deque``; the network is somebody else's job, 15 minutes later. +3. **It is off unless someone said yes.** The first gate is a cached level read, + and on an install that has not opted in that read returns ``off`` and the + function returns before it has looked at its own arguments. +4. **It cannot emit what the schema does not declare.** Properties are + allowlisted, type-checked and truncated by :mod:`condor.telemetry.schema`. + +The rate limiter is the fourth gate and exists for one specific failure: an +error loop turning the ``error`` event into a self-inflicted flood. Overflow +increments ``dropped``, which rides in the envelope — the collector sees an +honest count instead of a suspiciously quiet incident. +""" + +from __future__ import annotations + +import logging +import time +import uuid +from collections import deque + +log = logging.getLogger(__name__) + +RING_MAX = 2000 +RATE_PER_MIN = 60 +FLUSH_THRESHOLD = 200 + +_buffer: deque[dict] = deque(maxlen=RING_MAX) +_dropped = 0 +_tokens: dict[str, tuple[float, float]] = {} + +# False until a process registers a flush job (see condor.telemetry.init). A +# process without one — the out-of-process MCP server — writes straight to its +# own spool file instead, because an in-memory ring there would be lost on exit. +_hosted = False + + +def set_hosted(value: bool) -> None: + global _hosted + _hosted = value + + +def is_hosted() -> bool: + return _hosted + + +def _take_token(name: str) -> bool: + """Token bucket, :data:`RATE_PER_MIN` per event name per minute.""" + now = time.monotonic() + tokens, last = _tokens.get(name, (float(RATE_PER_MIN), now)) + tokens = min(float(RATE_PER_MIN), tokens + (now - last) * RATE_PER_MIN / 60.0) + if tokens < 1.0: + _tokens[name] = (tokens, now) + return False + _tokens[name] = (tokens - 1.0, now) + return True + + +def emit(name: str, /, **props) -> None: + """Record one event. Synchronous, never raises, never does I/O. + + The event name is positional-only so that an event may legitimately declare + a property called ``name`` (``command`` does) without colliding with it. + + Unknown event names and undeclared properties are dropped rather than + reported, so a stale call site degrades to silence instead of an exception + on a trading path. + """ + global _dropped + try: + from condor.telemetry import consent, schema + + level = consent.level() + if level == consent.OFF: + return + if not schema.allowed_at(name, level): + return + if not _take_token(name): + _dropped += 1 + return + clean = schema.sanitize(name, props) + if clean is None: + return + event = { + "id": uuid.uuid4().hex, + "ts": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), + "name": name, + "props": clean, + } + if _hosted: + if len(_buffer) == RING_MAX: + _dropped += 1 # the deque is about to evict the oldest + _buffer.append(event) + else: + from condor.telemetry import outbox + + outbox.spool(event) + except Exception: # noqa: BLE001 - the whole point of this function + try: + log.debug("Telemetry emit failed for %r", name, exc_info=True) + except Exception: + pass + + +def buffered() -> int: + return len(_buffer) + + +def dropped() -> int: + return _dropped + + +def should_flush() -> bool: + return len(_buffer) >= FLUSH_THRESHOLD + + +def drain() -> tuple[list[dict], int]: + """Take the ring and the dropped count together, and reset both.""" + global _dropped + events = list(_buffer) + _buffer.clear() + count, _dropped = _dropped, 0 + return events, count + + +def discard_buffer() -> None: + """Throw the ring away without sending it. Used when consent is denied.""" + global _dropped + _buffer.clear() + _dropped = 0 + _tokens.clear() + + +async def flush(reason: str = "job") -> int: + """Try to deliver everything pending. Returns the number of events sent. + + Returns 0 without reading a file when consent is not granted, and 0 without + touching the network when no endpoint is configured — the shipped state, in + which events simply accumulate in the capped outbox. + """ + try: + from condor.telemetry import consent, context, outbox + + if consent.state() != consent.GRANTED or consent.level() == consent.OFF: + return 0 + + events, dropped_count = drain() + events.extend(outbox.drain_spools()) + events.extend(outbox.take_stashed()) + if not events: + return 0 + + if not outbox.endpoint(): + # Nowhere to go yet (the collector is FEAT-024). Park them; the cap + # in outbox.py is what keeps this honest on a long-running install. + outbox.stash(events) + return 0 + + sent = 0 + for batch in outbox.batches(events): + envelope = context.envelope(batch, dropped_count, consent.level()) + dropped_count = 0 + if await outbox.post(envelope): + sent += len(batch) + else: + outbox.stash(batch) + return sent + except Exception: + log.debug("Telemetry flush failed (%s)", reason, exc_info=True) + return 0 diff --git a/condor/telemetry/outbox.py b/condor/telemetry/outbox.py new file mode 100644 index 000000000..a11b74871 --- /dev/null +++ b/condor/telemetry/outbox.py @@ -0,0 +1,231 @@ +"""Durable spool and the (deliberately inert) send path. + +Two files live under ``condor/.runtime/telemetry/``, the same gitignored place +the rest of the runtime keeps its append-only facts: + +``spool..jsonl`` + Written by processes that have no flush job of their own — in practice the + MCP server, which the agent spawns in its own process group with its own + interpreter and no ``job_queue``. Each process owns its own file, so there + is no locking and no interleaving; the host drains and deletes them. + +``outbox.jsonl`` + Where a batch goes when it could not be delivered. Capped at + :data:`MAX_OUTBOX_EVENTS` events and :data:`MAX_OUTBOX_AGE_S`, oldest first, + so an install that never reaches a collector accumulates a bounded file and + then quietly drops the excess. That is the intended behaviour, not a bug. + +**Nothing is transmitted unless an operator sets ``CONDOR_TELEMETRY_URL``.** No +collector address is compiled into this repository. With the variable unset — +which is the shipped state — :func:`post` returns ``False`` without importing a +network client, and every event's whole life is a local file that +:func:`purge` can delete. +""" + +from __future__ import annotations + +import json +import logging +import os +import time +from pathlib import Path + +from condor.fsutil import atomic_write_text + +log = logging.getLogger(__name__) + +MAX_OUTBOX_EVENTS = 5000 +MAX_OUTBOX_AGE_S = 7 * 24 * 3600 +MAX_BATCH_BYTES = 512 * 1024 +# Collectors cap an envelope by event count as well as by size — the reference +# collector refuses more than 500 with 413 too_many_events. Splitting on bytes +# alone let a backlog build envelopes no collector would ever accept, so the +# outbox could never drain. Keep this at or below the smallest cap we target. +MAX_BATCH_EVENTS = 500 +POST_TIMEOUT_S = 10 + + +def root() -> Path: + """Where the spool lives. Derived like every other runtime store.""" + from condor.agents.agent import _DATA_ROOT + + return Path(_DATA_ROOT).parent / "condor" / ".runtime" / "telemetry" + + +def spool_path(pid: int | None = None) -> Path: + return root() / f"spool.{pid or os.getpid()}.jsonl" + + +def outbox_path() -> Path: + return root() / "outbox.jsonl" + + +def _append(path: Path, records: list[dict]) -> None: + if not records: + return + path.parent.mkdir(parents=True, exist_ok=True) + with open(path, "a", encoding="utf-8") as fh: + for record in records: + fh.write(json.dumps(record, separators=(",", ":")) + "\n") + + +def _read(path: Path) -> list[dict]: + if not path.exists(): + return [] + events = [] + try: + with open(path, "r", encoding="utf-8") as fh: + for line in fh: + line = line.strip() + if not line: + continue + try: + events.append(json.loads(line)) + except ValueError: + continue # a torn last line from a killed process + except OSError: + log.debug("Telemetry could not read %s", path, exc_info=True) + return events + + +def spool(event: dict) -> None: + """Append one event to this process's own spool file.""" + _append(spool_path(), [event]) + + +def drain_spools() -> list[dict]: + """Take everything other processes left behind, including our own file. + + A spool file still being written by a live process is left alone unless it + is ours: deleting it would lose whatever landed between read and unlink. + """ + directory = root() + if not directory.is_dir(): + return [] + mine = os.getpid() + events: list[dict] = [] + for path in sorted(directory.glob("spool.*.jsonl")): + try: + pid = int(path.name.split(".")[1]) + except (IndexError, ValueError): + continue + if pid != mine and _alive(pid): + continue + events.extend(_read(path)) + try: + path.unlink() + except OSError: + pass + return events + + +def _alive(pid: int) -> bool: + try: + os.kill(pid, 0) + return True + except (ProcessLookupError, ValueError): + return False + except PermissionError: + return True + + +def stash(events: list[dict]) -> None: + """Park undeliverable events, then enforce the cap.""" + if not events: + return + _append(outbox_path(), events) + _trim() + + +def take_stashed() -> list[dict]: + """Read and clear the outbox, so a flush can retry it alongside fresh events.""" + path = outbox_path() + events = _read(path) + try: + path.unlink() + except OSError: + pass + return events + + +def _trim() -> None: + path = outbox_path() + events = _read(path) + cutoff = time.time() - MAX_OUTBOX_AGE_S + kept = [e for e in events if _epoch(e) >= cutoff][-MAX_OUTBOX_EVENTS:] + if len(kept) == len(events): + return + try: + atomic_write_text( + path, + "".join(json.dumps(r, separators=(",", ":")) + "\n" for r in kept), + ) + except OSError: + log.debug("Telemetry could not trim the outbox", exc_info=True) + + +def _epoch(event: dict) -> float: + try: + return time.mktime(time.strptime(event["ts"], "%Y-%m-%dT%H:%M:%SZ")) + except Exception: + return time.time() + + +def purge() -> None: + """Delete every trace. Called when consent is denied or withdrawn.""" + directory = root() + if not directory.is_dir(): + return + for path in list(directory.glob("*.jsonl")) + list(directory.glob("*.tmp")): + try: + path.unlink() + except OSError: + pass + try: + directory.rmdir() + except OSError: + pass + + +def batches(events: list[dict]) -> list[list[dict]]: + """Split into envelopes within both :data:`MAX_BATCH_BYTES` and + :data:`MAX_BATCH_EVENTS`.""" + out: list[list[dict]] = [] + current: list[dict] = [] + size = 0 + for event in events: + length = len(json.dumps(event, separators=(",", ":"))) + full = size + length > MAX_BATCH_BYTES or len(current) >= MAX_BATCH_EVENTS + if current and full: + out.append(current) + current, size = [], 0 + current.append(event) + size += length + if current: + out.append(current) + return out + + +def endpoint() -> str | None: + """The collector URL, or None — which is the shipped default.""" + from utils.config import CONDOR_TELEMETRY_URL + + return CONDOR_TELEMETRY_URL + + +async def post(envelope: dict) -> bool: + """Deliver one envelope. Returns False — without touching the network — when + no endpoint is configured, which is the state this repository ships in.""" + url = endpoint() + if not url: + return False + try: + import aiohttp + + timeout = aiohttp.ClientTimeout(total=POST_TIMEOUT_S) + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.post(url, json=envelope) as response: + return 200 <= response.status < 300 + except Exception: + log.debug("Telemetry POST failed; events stay in the outbox", exc_info=True) + return False diff --git a/condor/telemetry/prompt.py b/condor/telemetry/prompt.py new file mode 100644 index 000000000..dd3ee0957 --- /dev/null +++ b/condor/telemetry/prompt.py @@ -0,0 +1,127 @@ +"""The one-tap consent prompt, and the callback that answers it. + +Opt-in only works if asking is cheap, so this is a single message with three +buttons next to the "Condor is online" notification the admin already gets. It +is sent at most once per version, the intent is written to disk *before* the +message goes out (a crash loop must not re-ask forever), and until it is +answered the install emits nothing at all. +""" + +from __future__ import annotations + +import logging + +log = logging.getLogger(__name__) + +CALLBACK_PREFIX = "telemetry" + +_TEXT = ( + "Help improve Condor?\n\n" + "Condor can send an anonymous, allowlisted usage summary to the project: " + "which commands and screens get used, what breaks, and which models agents " + "run. It is off right now and stays off unless you say yes.\n\n" + "Never included: API keys, wallet addresses, server names or URLs, " + "trading pairs, amounts, balances, positions, prompts or agent replies, " + "and no Telegram id or username.\n\n" + "Full details in PRIVACY.md at the root of the repo, which also says how " + "to change or withdraw this answer at any time." +) + + +def keyboard(): + from telegram import InlineKeyboardButton, InlineKeyboardMarkup + + return InlineKeyboardMarkup( + [ + [ + InlineKeyboardButton( + "Yes, help improve Condor", callback_data=f"{CALLBACK_PREFIX}:usage" + ) + ], + [ + InlineKeyboardButton( + "Only count my install", callback_data=f"{CALLBACK_PREFIX}:ping" + ) + ], + [InlineKeyboardButton("No thanks", callback_data=f"{CALLBACK_PREFIX}:off")], + ] + ) + + +async def maybe_prompt_admin(bot) -> bool: + """Ask the admin once, if there is anything to ask. Never raises.""" + try: + from utils.config import ADMIN_USER_ID + + if not ADMIN_USER_ID: + return False + + from condor.telemetry import consent, context + + version = context.version() + if not consent.should_prompt(version): + return False + + # Written first: if sending or the process dies right after, the admin + # gets asked again on the next version, not on the next boot loop. + consent.mark_prompted(version) + await bot.send_message( + chat_id=int(ADMIN_USER_ID), text=_TEXT, reply_markup=keyboard() + ) + return True + except Exception: # noqa: BLE001 + log.debug("Could not send the telemetry consent prompt", exc_info=True) + return False + + +async def callback_handler(update, context) -> None: + """Handle ``telemetry:usage|ping|off``. Admin only — it is an install-wide + setting, and the admin owns the install.""" + query = update.callback_query + try: + await query.answer() + except Exception: # noqa: BLE001 + pass + + try: + from condor.telemetry import consent + from condor.telemetry.consent import _cm + + user = getattr(update, "effective_user", None) + cm = _cm() + if cm is None or user is None or not cm.is_admin(int(user.id)): + await query.edit_message_text( + "Only the admin can change the telemetry setting." + ) + return + + answer = ( + (query.data or "").split(":", 1)[1] if ":" in (query.data or "") else "" + ) + if answer == "off": + consent.deny() + await query.edit_message_text( + "Telemetry stays off. Nothing was collected, and the local " + "buffer has been deleted." + ) + return + + chosen = consent.grant(answer) + from condor.telemetry import emitter + + emitter.emit("install") + if chosen == consent.PING: + await query.edit_message_text( + "Thanks. Condor will only report that this install exists and " + "which version it runs. PRIVACY.md says how to change or " + "withdraw this." + ) + else: + await query.edit_message_text( + "Thanks. Condor will send anonymous usage and reliability " + "events. No keys, addresses, pairs, amounts or prompts ever " + "leave this machine. PRIVACY.md says how to change or withdraw " + "this." + ) + except Exception: # noqa: BLE001 + log.exception("Telemetry consent callback failed") diff --git a/condor/telemetry/schema.py b/condor/telemetry/schema.py new file mode 100644 index 000000000..3f8ce8481 --- /dev/null +++ b/condor/telemetry/schema.py @@ -0,0 +1,324 @@ +"""THE TAXONOMY — the allowlist that makes the privacy promise a property of code. + +Every event Condor can emit is declared here, with every property it is allowed +to carry and the shape that property must have. :func:`sanitize` is the only way +an event reaches a buffer, and it **drops** anything not declared rather than +passing it through. That is deliberate: the "never collected" list in +``PRIVACY.md`` is not a convention the call sites are trusted to honour, it is +what this module structurally cannot let past. + +Consequences worth stating out loud, because they are the point: + +- An undeclared property is discarded, whatever it is called. A tap that grows a + ``wallet=`` argument by accident leaks nothing. +- Free-form strings are capped at :data:`MAX_STR` characters, so nothing long + enough to be a key, an address, a prompt or a URL survives intact. +- Enum properties are snapped to their allowlist or to ``other``, so cardinality + is bounded by this file rather than by whatever the caller happened to hold. + +Deliberately absent, and asserted by ``tests/test_telemetry.py``: amounts, +balances, PnL, trading pairs, order ids, wallet addresses, server names, URLs, +hostnames, IPs, API keys, Telegram ids or usernames, prompts, agent replies, +routine configs, journal text, and exception message strings. +""" + +from __future__ import annotations + +import re +from dataclasses import dataclass, field + +# One place to change the blast radius of any single value. +MAX_STR = 64 +MAX_MAP_KEYS = 20 +MAX_LIST = 5 + +# Free-form strings keep only characters our own identifiers use. It is a second +# fence behind the allowlist: even a declared string property cannot carry a +# sentence, a path with a home directory in it, or a URL with a token in it. +_SAFE_CHARS = re.compile(r"[^A-Za-z0-9._:/@#-]+") + + +@dataclass(frozen=True) +class PropSpec: + """What one property of one event is allowed to be.""" + + kind: str # str | int | float | bool | enum | map | list + allowed: tuple[str, ...] = field(default=()) + + def __post_init__(self) -> None: + if self.kind == "enum" and not self.allowed: + raise ValueError("enum PropSpec needs an allowlist") + + +def _s() -> PropSpec: + return PropSpec("str") + + +def _i() -> PropSpec: + return PropSpec("int") + + +def _f() -> PropSpec: + return PropSpec("float") + + +def _b() -> PropSpec: + return PropSpec("bool") + + +def _e(*allowed: str) -> PropSpec: + return PropSpec("enum", tuple(allowed)) + + +SURFACES = ("telegram", "web", "mcp", "strategy", "other") + +# The commands we ship. Anything else a user types is reported as `other`, so a +# typo, a third-party bot's command, or a private fork's addition can never +# widen the value space. +COMMANDS = ( + "start", + "portfolio", + "bots", + "new_bot", + "trade", + "swap", + "lp", + "routines", + "executors", + "agent", + "stop", + "delegations", + "memory", + "cancel", + "servers", + "keys", + "gateway", + "admin", + "update", + "web", +) + +MODULES = ( + "bots", + "cex", + "dex", + "trade", + "agents", + "routines", + "config", + "portfolio", + "executors", + "admin", + "start", + "other", +) + +# Events an install at level `ping` may send. Everything else needs `usage`. +PING_EVENTS = frozenset({"install", "heartbeat", "version_change", "shutdown"}) + + +EVENTS: dict[str, dict[str, PropSpec]] = { + # ── Adoption & retention ───────────────────────────────────────────── + # `install` carries no properties: the envelope context is the payload. + "install": {}, + "heartbeat": { + "uptime_h": _f(), + "active_users_24h": _i(), + "surfaces": PropSpec("map"), + # `health` rides here rather than being its own event. + "hb_api_online": _b(), + "gateway_online": _b(), + "hb_api_version": _s(), + }, + "version_change": { + # Named `from_version`/`to_version`, not `from`/`to`: `from` is a Python + # keyword and emit() takes keyword arguments. + "from_version": _s(), + "to_version": _s(), + "was_behind": _i(), + }, + "shutdown": { + "reason": _e("signal", "crash", "restart"), + "uptime_h": _f(), + }, + # ── Feature usage ──────────────────────────────────────────────────── + "command": { + "name": _e(*COMMANDS, "other"), + "surface": _e(*SURFACES), + "user_hash": _s(), + "authorized": _b(), + }, + "action": { + "module": _e(*MODULES), + "verb": _s(), + "surface": _e(*SURFACES), + "status": _i(), + }, + "feature_first_use": {"feature": _s()}, + "bot_deploy": { + "controller_type": _s(), + "connector": _s(), + "is_paper": _b(), + }, + "executor_deploy": {"executor_type": _s(), "connector": _s()}, + "trade": { + # No trading pair, and no amount. See PRIVACY.md: an install that trades + # one pair, plus timestamps, is a deanonymizable position disclosure. + "venue": _e("cex", "dex"), + "connector": _s(), + "side": _e("buy", "sell", "other"), + "order_type": _s(), + }, + "routine_run": { + "routine": _s(), + "kind": _e("oneshot", "continuous", "other"), + "trigger": _e("manual", "schedule", "background", "web", "mcp", "other"), + "duration_ms": _i(), + "ok": _b(), + }, + # ── Reliability ────────────────────────────────────────────────────── + "error": { + "where": _s(), + "exc_type": _s(), + # A hash of the message, never the message. Grouping still works; the + # balance, hostname or key that message might have contained does not + # survive the hash. + "sig": _s(), + "frames": PropSpec("list"), + "surface": _e(*SURFACES), + "fatal": _b(), + }, + "upstream_error": { + "service": _e("hb_api", "gateway", "llm", "telegram", "other"), + "op": _s(), + "status": _s(), + }, + # ── Agent economics ────────────────────────────────────────────────── + "agent_turn": { + "kind": _e("chat", "consult", "delegate", "tick", "other"), + "provider": _s(), + "model": _s(), + "tool_calls": _i(), + "duration_ms": _i(), + "outcome": _e("done", "error", "aborted"), + "tokens_in": _i(), + "tokens_out": _i(), + "surface": _e(*SURFACES), + # `agent_tools` rides here: a bounded map of tool name -> count. + "tools": PropSpec("map"), + }, + # An MCP tool call observed from the out-of-process server, which cannot see + # the turn it belongs to (see condor/telemetry/outbox.py). Not in the + # original design's table; added so the subprocess spool has something + # declared to carry. + "mcp_tool": {"tool": _s(), "ok": _b(), "duration_ms": _i()}, + "strategy_run": { + "mode": _e("dry_run", "run_once", "loop", "other"), + "frequency_sec": _i(), + "ticks": _i(), + "bot_mode": _s(), + "has_risk_limits": _b(), + "stopped_by": _s(), + }, + "confirmation": { + "tool": _s(), + "decision": _e("allow", "deny", "timeout"), + }, +} + + +def is_known(name: str) -> bool: + return name in EVENTS + + +def allowed_at(name: str, level: str) -> bool: + """Is this event permitted at this telemetry level?""" + if level == "usage": + return name in EVENTS + if level == "ping": + return name in PING_EVENTS + return False + + +def clean_str(value: object) -> str | None: + """Reduce any value to a short, identifier-shaped string, or reject it.""" + if isinstance(value, bool) or not isinstance(value, (str, int, float)): + return None + text = _SAFE_CHARS.sub("_", str(value)).strip("_") + if not text: + return None + return text[:MAX_STR] + + +def _coerce(spec: PropSpec, value: object) -> object | None: + """Force one value into its declared shape, or return None to drop it.""" + if value is None: + return None + + if spec.kind == "bool": + return value if isinstance(value, bool) else None + + if spec.kind == "int": + # bool is an int in Python; a bool in an int slot is a caller bug. + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return int(value) + + if spec.kind == "float": + if isinstance(value, bool) or not isinstance(value, (int, float)): + return None + return round(float(value), 3) + + if spec.kind == "str": + return clean_str(value) + + if spec.kind == "enum": + text = clean_str(value) + if text is None: + return None + return text if text in spec.allowed else "other" + + if spec.kind == "list": + if not isinstance(value, (list, tuple)): + return None + out = [s for s in (clean_str(v) for v in value[:MAX_LIST]) if s] + return out or None + + if spec.kind == "map": + if not isinstance(value, dict): + return None + out: dict[str, int] = {} + for key, count in list(value.items())[:MAX_MAP_KEYS]: + safe_key = clean_str(key) + if safe_key is None: + continue + if isinstance(count, bool) or not isinstance(count, (int, float)): + continue + out[safe_key] = int(count) + return out or None + + return None + + +def sanitize(name: str, props: dict | None) -> dict | None: + """Return the emittable form of one event, or None if it is not emittable. + + Undeclared properties are dropped silently — the caller is not trusted, and + a warning that names the offending value would only move the leak into the + log file. + """ + spec = EVENTS.get(name) + if spec is None: + return None + if not props: + return {} + + clean: dict = {} + for key, value in props.items(): + prop_spec = spec.get(key) + if prop_spec is None: + continue + coerced = _coerce(prop_spec, value) + if coerced is not None: + clean[key] = coerced + return clean diff --git a/condor/telemetry/taps.py b/condor/telemetry/taps.py new file mode 100644 index 000000000..c32117e55 --- /dev/null +++ b/condor/telemetry/taps.py @@ -0,0 +1,446 @@ +"""The taps: the only code that turns something happening into an event. + +Call sites stay one line long and stay dumb. Everything that decides *what* is +allowed to be said lives here and in :mod:`condor.telemetry.schema`, so the +privacy review has two files to read rather than fifteen. + +Every public function here is defensive to the point of paranoia: a tap runs +inside a command handler, an error handler, an order path and a tick loop, and +a tap that raises would turn "we learned nothing" into "the bot broke". They all +swallow, and :func:`condor.telemetry.emitter.emit` swallows again underneath. +""" + +from __future__ import annotations + +import functools +import hashlib +import logging +import re +import time +import traceback +from pathlib import Path + +from condor.telemetry.emitter import emit + +log = logging.getLogger(__name__) + +TELEMETRY_FLUSH_JOB = "telemetry_flush" +TELEMETRY_HEARTBEAT_JOB = "telemetry_heartbeat" +FLUSH_INTERVAL_S = 15 * 60 +HEARTBEAT_INTERVAL_S = 6 * 3600 + +_REPO = Path(__file__).resolve().parent.parent.parent +# Frames from our own packages only. A frame in site-packages tells us nothing +# actionable and its path can carry a username. +_OWN_PACKAGES = ("condor", "handlers", "utils", "mcp_servers", "routines") + +# Rolling, in-memory only, never persisted and never transmitted as a set — +# only its cardinality reaches an envelope. +_seen_users: dict[str, float] = {} +_surface_counts: dict[str, int] = {} + +# A leading run of lowercase and underscores. Deliberately strict: callback data +# like ``admin:approve_12345`` or ``bots:view_MyBotName`` carries an identifier +# after the verb, and cutting at the first digit or capital drops it. +_VERB = re.compile(r"^[a-z][a-z_]{0,31}") + + +def _quiet(fn): + """A tap must never be the reason a handler failed.""" + + @functools.wraps(fn) + def wrapper(*args, **kwargs): + try: + return fn(*args, **kwargs) + except Exception: # noqa: BLE001 + log.debug("Telemetry tap %s failed", fn.__name__, exc_info=True) + return None + + return wrapper + + +def _on() -> bool: + from condor.telemetry import consent + + return consent.level() != consent.OFF + + +# ── Identity ───────────────────────────────────────────────────────────── + + +@_quiet +def user_hash(user_id) -> str | None: + """A per-install pseudonym for one Telegram user. + + Salted with the install's own secret, which never leaves the machine, so the + result counts distinct users within an install and is useless for correlating + the same person across two installs. The Telegram id itself is never sent. + """ + from condor.telemetry import consent + + secret = consent.install_secret() + if not secret or user_id is None: + return None + digest = hashlib.sha256(f"{secret}:{user_id}".encode()).hexdigest() + return digest[:16] + + +def _note_user(hashed: str | None, surface: str) -> None: + if hashed: + _seen_users[hashed] = time.time() + _surface_counts[surface] = _surface_counts.get(surface, 0) + 1 + + +@_quiet +def feature_first_use(feature: str) -> None: + """Fire once per install, ever. The activation funnel.""" + if not _on(): + return + from condor.telemetry import consent + + if consent.mark_feature_seen(feature): + emit("feature_first_use", feature=feature) + + +def _verb_of(segment: str) -> str: + match = _VERB.match((segment or "").strip()) + return (match.group(0).rstrip("_") if match else "") or "other" + + +# ── Telegram ───────────────────────────────────────────────────────────── + + +async def telegram_tap(update, context) -> None: + """Observe every update, in ``group=-1``, without ever deciding anything. + + Registered before the real handlers so it sees commands and callbacks from + unauthorized users too — who is knocking is signal. It reads authorization + state; it must never call into ``@restricted``, or a read would become an + access-control side effect. + """ + try: + if not _on(): + return + user = getattr(update, "effective_user", None) + hashed = user_hash(getattr(user, "id", None)) if user else None + authorized = _is_authorized(getattr(user, "id", None)) + + message = getattr(update, "message", None) + text = (getattr(message, "text", None) or "") if message else "" + if text.startswith("/"): + raw = text[1:].split(maxsplit=1)[0].split("@")[0].lower() + _note_user(hashed, "telegram") + emit( + "command", + name=raw, + surface="telegram", + user_hash=hashed, + authorized=authorized, + ) + if authorized: + feature_first_use(f"command:{_verb_of(raw)}") + return + + query = getattr(update, "callback_query", None) + data = (getattr(query, "data", None) or "") if query else "" + if data: + module, _, rest = data.partition(":") + _note_user(hashed, "telegram") + emit( + "action", + module=module.lower(), + verb=_verb_of(rest.split(":")[0]), + surface="telegram", + ) + except Exception: # noqa: BLE001 + log.debug("Telegram telemetry tap failed", exc_info=True) + + +@_quiet +def _is_authorized(user_id) -> bool: + from condor.telemetry.consent import _cm + + cm = _cm() + return bool(cm and user_id is not None and cm.is_approved(int(user_id))) + + +# ── Errors ─────────────────────────────────────────────────────────────── + + +def _frames(exc: BaseException) -> list[str]: + """Up to five ``file:line`` pairs from our own packages, repo-relative. + + Absolute paths are not sent: they contain the operator's home directory and + often their username. + """ + out: list[str] = [] + for frame in traceback.extract_tb(exc.__traceback__): + try: + path = Path(frame.filename).resolve().relative_to(_REPO) + except (ValueError, OSError): + continue + if path.parts and path.parts[0] in _OWN_PACKAGES or path.name == "main.py": + out.append(f"{path.as_posix()}:{frame.lineno}") + return out[-5:] + + +@_quiet +def on_error( + exc: BaseException | None, where: str, surface: str = "other", fatal: bool = False +) -> None: + """Report that something broke — the shape of it, never its words. + + Only the exception *type*, a hash of its message, and our own stack frames. + Message strings are where balances, hostnames, URLs and keys leak, so they + are hashed and discarded: grouping still works, disclosure does not. + """ + if exc is None or not _on(): + return + signature = hashlib.sha256(str(exc).encode("utf-8", "replace")).hexdigest()[:12] + emit( + "error", + where=where, + exc_type=type(exc).__name__, + sig=signature, + frames=_frames(exc), + surface=surface, + fatal=fatal, + ) + + +@_quiet +def on_upstream_error(service: str, op: str, status) -> None: + """A dependency answered badly: which one, which operation, which code.""" + emit("upstream_error", service=service, op=op, status=str(status)) + + +# ── Web ────────────────────────────────────────────────────────────────── + + +def _route_action(request) -> tuple[str, str] | None: + """Turn a matched route *template* into (module, verb). + + The template, never the URL: ``/api/v1/bots/{name}`` rather than + ``/api/v1/bots/my-arb-bot``. Cardinality is bounded by our own router, and + no path parameter — a bot name, a server name, a report id — is ever read. + """ + route = request.scope.get("route") + template = getattr(route, "path", None) + if not template: + return None + parts = [p for p in template.strip("/").split("/") if p] + if parts[:2] == ["api", "v1"]: + parts = parts[2:] + if not parts: + return None + module = parts[0].lower() + static = [p for p in parts[1:] if not p.startswith("{")] + verb = _verb_of("_".join(static)) if static else request.method.lower() + return module, verb + + +async def web_tap(request, call_next): + """ASGI middleware: one ``action`` per API call, with its status code. + + Deliberately not wrapped in a swallow-everything decorator: ``call_next`` + must be allowed to raise, or an unhandled route exception would surface as + "no response returned" instead of the 500 it is. Only the observation is + guarded. + """ + response = await call_next(request) + try: + if _on() and str(request.url.path).startswith("/api/"): + action = _route_action(request) + if action: + module, verb = action + _surface_counts["web"] = _surface_counts.get("web", 0) + 1 + emit( + "action", + module=module, + verb=verb, + surface="web", + status=response.status_code, + ) + except Exception: # noqa: BLE001 + log.debug("Web telemetry tap failed", exc_info=True) + return response + + +# ── Agents, strategies, routines, confirmations ────────────────────────── + + +@_quiet +def agent_turn( + *, + kind: str, + provider: str = "", + model: str = "", + tool_calls: int = 0, + duration_ms: int = 0, + outcome: str = "done", + surface: str = "other", + tools: dict | None = None, +) -> None: + """One turn through ``condor.runtime.client.prompt`` — the single funnel + Telegram, the dashboard and MCP all cross.""" + _surface_counts[surface] = _surface_counts.get(surface, 0) + 1 + emit( + "agent_turn", + kind=kind, + provider=provider, + model=model, + tool_calls=tool_calls, + duration_ms=duration_ms, + outcome=outcome, + surface=surface, + tools=tools, + ) + + +@_quiet +def strategy_run(config: dict | None, *, ticks: int = 0, stopped_by: str = "") -> None: + """A tick-engine session ended. Shape of the run only — never its journal.""" + config = config or {} + emit( + "strategy_run", + mode=config.get("execution_mode", "loop"), + frequency_sec=config.get("frequency_sec"), + ticks=ticks, + bot_mode=config.get("bot_mode"), + has_risk_limits=bool(config.get("risk_limits")), + stopped_by=stopped_by, + ) + + +@_quiet +def routine_run( + *, routine: str, kind: str, trigger: str, duration_ms: int, ok: bool +) -> None: + """A routine ran. ``routine`` is the shipped name, or ``custom`` when the + file is not one of ours — a user's private routine name is theirs.""" + emit( + "routine_run", + routine=routine, + kind=kind, + trigger=trigger, + duration_ms=duration_ms, + ok=ok, + ) + + +def shipped_routine_name(name: str) -> str: + """``name`` if the routine ships in this repo, else ``custom``.""" + try: + if (_REPO / "routines" / f"{name}.py").exists(): + return name + except Exception: # noqa: BLE001 + pass + return "custom" + + +@_quiet +def confirmation(tool: str, decision: str) -> None: + emit("confirmation", tool=tool, decision=decision) + + +@_quiet +def version_change(from_version: str, to_version: str, was_behind: int = 0) -> None: + emit( + "version_change", + from_version=from_version, + to_version=to_version, + was_behind=was_behind, + ) + + +def tracked(tool_name: str): + """Decorator for an MCP tool, which runs in its own process. + + That process has no ``job_queue`` and no shared heap, so the event goes + straight to its own ``spool..jsonl`` and the host drains it later. Only + the tool name, whether it worked, and how long it took — never arguments, + never results. + """ + + def decorator(fn): + @functools.wraps(fn) + async def wrapper(*args, **kwargs): + started = time.monotonic() + ok = True + try: + return await fn(*args, **kwargs) + except Exception: + ok = False + raise + finally: + emit( + "mcp_tool", + tool=tool_name, + ok=ok, + duration_ms=int((time.monotonic() - started) * 1000), + ) + + return wrapper + + return decorator + + +# ── Jobs ───────────────────────────────────────────────────────────────── + + +async def flush_job(context) -> None: + """The only place in the process that may talk to a collector.""" + from condor.telemetry import emitter + + await emitter.flush("job") + + +async def heartbeat_job(context) -> None: + """Proof of life from an install that did nothing else this cycle.""" + try: + if not _on(): + return + from condor.telemetry import context as ctx + + cutoff = time.time() - 24 * 3600 + for hashed, seen in list(_seen_users.items()): + if seen < cutoff: + _seen_users.pop(hashed, None) + surfaces = dict(_surface_counts) + _surface_counts.clear() + emit( + "heartbeat", + uptime_h=ctx.uptime_h(), + active_users_24h=len(_seen_users), + surfaces=surfaces, + ) + except Exception: # noqa: BLE001 + log.debug("Heartbeat tap failed", exc_info=True) + + +def register_jobs(application) -> None: + """Register the flush and heartbeat jobs, the house pattern from + ``handlers.admin.update.schedule_update_checks``. + + Registered unconditionally: the jobs are cheap, and both return immediately + when consent is absent. That keeps a mid-run opt-in working without a + restart. + """ + try: + queue = getattr(application, "job_queue", None) + if queue is None: + return + for name in (TELEMETRY_FLUSH_JOB, TELEMETRY_HEARTBEAT_JOB): + for job in queue.get_jobs_by_name(name): + job.schedule_removal() + queue.run_repeating( + flush_job, interval=FLUSH_INTERVAL_S, first=60, name=TELEMETRY_FLUSH_JOB + ) + queue.run_repeating( + heartbeat_job, + interval=HEARTBEAT_INTERVAL_S, + first=120, + name=TELEMETRY_HEARTBEAT_JOB, + ) + except Exception: # noqa: BLE001 + log.debug("Could not register telemetry jobs", exc_info=True) diff --git a/condor/web/app.py b/condor/web/app.py index e5da32c28..e5ca6d623 100644 --- a/condor/web/app.py +++ b/condor/web/app.py @@ -9,18 +9,23 @@ from fastapi.responses import FileResponse from fastapi.staticfiles import StaticFiles +from condor.telemetry.taps import web_tap from condor.web.routes import ( + admin, agents, archived, auth, backtesting, bots, chat_ws, + code, confirmations, controller_performance, conversations, + dex, executors, market, + meta, portfolio, positions, reports, @@ -62,8 +67,15 @@ def create_app() -> FastAPI: allow_headers=["*"], ) + # Usage telemetry (FEAT-023). Reports the matched route *template* + # (`/api/v1/bots/{name}`), never the URL, so cardinality is bounded by our + # own router and no path parameter — a bot name, a server name, a report id + # — is ever read. No-op unless the admin opted in. + app.middleware("http")(web_tap) + # ── API routes ── app.include_router(auth.router, prefix="/api/v1") + app.include_router(admin.router, prefix="/api/v1") app.include_router(servers.router, prefix="/api/v1") app.include_router(portfolio.router, prefix="/api/v1") app.include_router(bots.router, prefix="/api/v1") @@ -73,9 +85,12 @@ def create_app() -> FastAPI: app.include_router(positions.router, prefix="/api/v1") app.include_router(backtesting.router, prefix="/api/v1") app.include_router(market.router, prefix="/api/v1") + app.include_router(dex.router, prefix="/api/v1") + app.include_router(meta.router, prefix="/api/v1") app.include_router(ws.router, prefix="/api/v1") app.include_router(agents.router, prefix="/api/v1") app.include_router(routines.router, prefix="/api/v1") + app.include_router(code.router, prefix="/api/v1") app.include_router(reports.router, prefix="/api/v1") app.include_router(settings.router, prefix="/api/v1") app.include_router(sessions.router, prefix="/api/v1") diff --git a/condor/web/auth.py b/condor/web/auth.py index 774ae85e7..e2986ee01 100644 --- a/condor/web/auth.py +++ b/condor/web/auth.py @@ -8,7 +8,7 @@ import time from typing import Optional -from fastapi import Depends, HTTPException, WebSocket, status +from fastapi import Depends, HTTPException, Query, WebSocket, status from fastapi.security import HTTPAuthorizationCredentials, HTTPBearer from jose import JWTError, jwt @@ -154,6 +154,59 @@ async def get_current_user( ) +# ── Server-scoped access (SEC-147) ── + + +def check_server_access(user_id: int, server_name: str) -> None: + """Raise ``403 No access`` unless ``user_id`` may use ``server_name``. + + Single implementation of the guard that used to be hand-copied into every + server-scoped web endpoint. ``has_server_access`` defaults to + :attr:`ServerPermission.TRADER`, which is the level every web call already + required. Endpoints that need a *stronger* level (owner-only credential and + server mutations in ``routes/settings.py``) still layer their own check on + top of this one — this is the floor, never the ceiling. + + Endpoints whose server name arrives in the request **body** call this + directly (a path/query dependency cannot see the body); endpoints that take + it as a path or query parameter use the ``require_server_access*`` + dependencies below, which are thin wrappers over this function. + """ + if not get_config_manager().has_server_access(user_id, server_name): + raise HTTPException(status_code=status.HTTP_403_FORBIDDEN, detail="No access") + + +async def require_server_access( + name: str, user: WebUser = Depends(get_current_user) +) -> WebUser: + """Auth dependency for endpoints routed under ``/servers/{name}/...``. + + Drop-in replacement for ``Depends(get_current_user)``: it returns the same + :class:`WebUser`, having first enforced access to the ``{name}`` path + parameter. If the route has no ``{name}`` path parameter FastAPI treats it + as a required *query* parameter and the request fails with 422 — the guard + fails closed, never open. + """ + check_server_access(user.id, name) + return user + + +async def require_server_access_by_server_name( + server_name: str, user: WebUser = Depends(get_current_user) +) -> WebUser: + """Same as :func:`require_server_access` for a ``{server_name}`` path param.""" + check_server_access(user.id, server_name) + return user + + +async def require_server_access_query( + server: str = Query(...), user: WebUser = Depends(get_current_user) +) -> WebUser: + """Same as :func:`require_server_access` for a ``?server=`` query param.""" + check_server_access(user.id, server) + return user + + # ── One-time login tokens (generated from Telegram /web command) ── diff --git a/condor/web/models.py b/condor/web/models.py index 107f5da48..757a51a24 100644 --- a/condor/web/models.py +++ b/condor/web/models.py @@ -496,7 +496,7 @@ class UpdateServerRequest(BaseModel): class GatewayStartRequest(BaseModel): # The Hummingbot API always runs the Gateway secured (TLS + mTLS) and manages the # certificates/passphrase itself (hummingbot-api SEC-048), so only image/port are sent. - image: str = "hummingbot/gateway:latest" + image: str = "hummingbot/gateway:development" port: int = 15888 @@ -506,7 +506,24 @@ class CredentialInfo(BaseModel): class GatewayPullRequest(BaseModel): - image: str = "hummingbot/gateway:latest" + image: str = "hummingbot/gateway:development" + + +class GatewayNetworkUpdateRequest(BaseModel): + # Partial network config (snake_case keys, e.g. {"node_url": "https://..."}). + # The Gateway validates values against its own JSON schema. + config: dict[str, Any] + + +class GatewayWalletAddRequest(BaseModel): + chain: str + private_key: str + set_default: bool = False + + +class GatewayWalletDefaultRequest(BaseModel): + chain: str + address: str class AddCredentialRequest(BaseModel): diff --git a/condor/web/routes/_errors.py b/condor/web/routes/_errors.py new file mode 100644 index 000000000..0bdd4c2c2 --- /dev/null +++ b/condor/web/routes/_errors.py @@ -0,0 +1,40 @@ +"""One mapping from a failed upstream call to a response that leaks nothing. + +``str(exc)`` on an ``aiohttp`` client exception embeds the backend's own URL, so +handing it to a browser as an ``HTTPException`` detail publishes the internal +host and port to anyone who can provoke a backend blip — a timeout, a refused +connection, an upstream 5xx. SEC-116 established the remedy for the executor +mutations and SEC-126 extended it to the executor reads; this module is that +same remedy lifted out of ``routes/executors.py`` so every route module shares +one implementation instead of growing a second sanitizer. + +The description itself still comes from :func:`condor.fetchers.describe_executor_error`, +which reads the safe pieces off the exception's attributes (``status`` is the +code the API answered with, ``message`` is the API's own ``detail``) and +collapses a transport failure — which has neither — to a generic line. +""" + +from __future__ import annotations + +from fastapi import HTTPException + +from condor.fetchers.executors import describe_executor_error + +__all__ = ["upstream_error"] + + +def upstream_error(action: str, exc: Exception) -> HTTPException: + """Map a failed backend call to an ``HTTPException`` that leaks nothing. + + An upstream 4xx is the caller's own bad request and stays a 400; anything + else — an upstream 5xx, a timeout, a refused connection — is this gateway + failing to reach its backend, so 502. The detail carries the API's message + but never the raw exception. + + Callers log the exception before raising: the address belongs in the server + log, where an operator needs it, not in the response, where a trader on a + shared server would read it. + """ + status, message = describe_executor_error(exc) + code = 400 if status is not None and 400 <= status < 500 else 502 + return HTTPException(status_code=code, detail=f"{action}: {message}") diff --git a/condor/web/routes/admin.py b/condor/web/routes/admin.py new file mode 100644 index 000000000..3272a2597 --- /dev/null +++ b/condor/web/routes/admin.py @@ -0,0 +1,119 @@ +"""Admin-only user administration for the web dashboard (ARCH-177). + +Today this carries exactly one capability, the ``code_run`` grant SEC-151 +introduced: the gate in ``routes/code.py`` is *admin or an explicit per-user +grant*, and until now the only way to set that grant was hand-editing +config.yml. This is where an admin sets it instead. + +Every route here is admin-only, checked server-side by ``_require_admin``. The +dashboard hides the panel from non-admins, but that is cosmetic — hiding a +control is not a gate, so each handler asks again and answers 403 whatever the +client believes it is. +""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from condor.web.auth import get_current_user +from condor.web.models import WebUser +from config_manager import UserRole, get_config_manager + +log = logging.getLogger(__name__) +router = APIRouter(prefix="/admin", tags=["admin"]) + + +def _require_admin(user: WebUser) -> None: + """Refuse anyone who is not a Condor admin. + + Mirrors the shape of the telemetry gate in ``routes/settings.py``: the role + is re-read from the ConfigManager rather than trusted from the JWT claim, + so demoting a user takes effect on their next request. + """ + if not get_config_manager().is_admin(user.id): + raise HTTPException( + status_code=403, + detail="Only an admin can administer users", + ) + + +class AdminUser(BaseModel): + """A user row as the admin panel needs it.""" + + user_id: int + username: str = "" + role: str = "" + # Whether the explicit grant is set on this record. Admins are reported + # separately via `is_admin`: they can already run code without a grant, and + # showing them as "granted" would misrepresent what revoking would do. + code_run: bool = False + is_admin: bool = False + + +@router.get("/users", response_model=list[AdminUser]) +async def list_users(user: WebUser = Depends(get_current_user)): + """Every user record, with the code_run grant each one carries.""" + _require_admin(user) + cm = get_config_manager() + rows = [] + for record in cm.get_all_users(): + uid = record.get("user_id") + if uid is None: + continue + rows.append( + AdminUser( + user_id=uid, + username=record.get("username") or "", + role=record.get("role") or "", + code_run=cm.has_code_run_grant(uid), + is_admin=record.get("role") == UserRole.ADMIN.value, + ) + ) + rows.sort(key=lambda r: (not r.is_admin, r.username.lower(), r.user_id)) + return rows + + +class CodeRunGrantRequest(BaseModel): + granted: bool + + +@router.put("/users/{user_id}/code-run") +async def set_code_run_grant( + user_id: int, + body: CodeRunGrantRequest, + user: WebUser = Depends(get_current_user), +): + """Grant or revoke ``code_run`` for one user. + + The grant confers arbitrary Python execution inside the bot process — it is + admin-equivalent in practice — so it is admin-only, audited by + ``set_code_run_grant``, and takes effect on the next request without a + restart because the gate re-reads config.yml every time. + + Granting to an admin is refused rather than silently stored: they already + pass the gate on the admin arm, so the grant would be a no-op record that + survives a demotion and quietly outlives the role that justified it. + """ + _require_admin(user) + cm = get_config_manager() + + if cm.get_user(user_id) is None: + raise HTTPException(status_code=404, detail=f"Unknown user {user_id}") + if body.granted and cm.is_admin(user_id): + raise HTTPException( + status_code=409, + detail="Admins can already run code; no grant is needed", + ) + + changed = cm.set_code_run_grant(user_id, body.granted, admin_id=user.id) + if changed: + log.warning( + "Admin %s %s code_run for user %s", + user.id, + "granted" if body.granted else "revoked", + user_id, + ) + return {"user_id": user_id, "code_run": cm.has_code_run_grant(user_id)} diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index 122903518..66753b4e2 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -39,7 +39,7 @@ list_session_snapshots, list_sessions, ) -from condor.web.auth import get_current_user +from condor.web.auth import check_server_access, get_current_user from condor.web.models import ReportSummary, WebUser # ── Simple in-memory TTL cache for performance data ── @@ -101,6 +101,7 @@ class RunningInstance(BaseModel): total_amount_quote: float = 100.0 trading_context: str = "" frequency_sec: int = 60 + tick_timeout_sec: int = 600 execution_mode: str = "loop" risk_limits: dict[str, Any] = {} @@ -777,6 +778,7 @@ def _instance_from_engine(engine, perf_by_id: dict) -> RunningInstance: total_amount_quote=info.get("total_amount_quote", 100), trading_context=info.get("trading_context", ""), frequency_sec=info.get("frequency_sec", 60), + tick_timeout_sec=info.get("tick_timeout_sec", 600), agent_key=info.get("agent_key", ""), execution_mode=info.get("execution_mode", "loop"), risk_limits=info.get("risk_limits", {}), @@ -1166,10 +1168,8 @@ async def update_agent_config( # A pin decides which account the Agent's tools trade on, so it is gated # like every other server-scoped write. An empty string clears the pin and # needs no access at all. - if req.server_name and not get_config_manager().has_server_access( - user.id, req.server_name - ): - raise HTTPException(status_code=403, detail="No access") + if req.server_name: + check_server_access(user.id, req.server_name) # Picker sentinels ("openrouter:", "custom:") are drill-downs that open a # model list, not startable models: stored here they would fail at every @@ -1226,10 +1226,8 @@ async def consult_agent( # routes do — otherwise any session could consult against a server it was # never granted (IDOR). Only enforce when a server is actually requested; # serverless consults need no server scope. - if req.server_name and not get_config_manager().has_server_access( - user.id, req.server_name - ): - raise HTTPException(status_code=403, detail="No access") + if req.server_name: + check_server_access(user.id, req.server_name) # Web callers always act as themselves; the ``user_id`` override is reserved # for trusted internal/MCP callers and must not let a session impersonate @@ -1286,10 +1284,8 @@ async def delegate_agent( # Same server-scope gate as consult: a delegate binds the agent's MCP toolset # to ``server_name``'s live credentials, so refuse a server the caller can't access. - if req.server_name and not get_config_manager().has_server_access( - user.id, req.server_name - ): - raise HTTPException(status_code=403, detail="No access") + if req.server_name: + check_server_access(user.id, req.server_name) conversation_id = await _conversation_for_session(req.session_key) @@ -1737,11 +1733,36 @@ async def _start(agent, strategy, req: StartStrategyRequest, user_id: int) -> di """Spawn a TickEngine session for ``strategy`` under ``agent``.""" from condor.agents.config import load_full_config from condor.agents.engine import TickEngine + from config_manager import get_config_manager config_dict = load_full_config(strategy.dir, strategy.default_config) if req.config: config_dict.update(req.config) + # ``TickEngine._resolve_server`` trades on ``config["server_name"]`` and the + # request body is a free-form dict, so without this gate any authenticated + # user could start a live loop on another user's stored credentials — the + # same check the config pin and consult/delegate already apply. A name the + # body asked for is held to it strictly; an inherited one (strategy default, + # or the "local" that AgentConfig fills in) only matters when it resolves to + # a real server, since otherwise the engine falls through to the caller's + # own accessible servers, which is scoped already. + cm = get_config_manager() + server_name = config_dict.get("server_name") + asked_for_it = bool(req.config and req.config.get("server_name")) + if server_name and (asked_for_it or cm.get_server(server_name)): + check_server_access(user_id, server_name) + # A name the body asked for must also name something. An unknown one + # used to be waved through on the reasoning that it borrows no + # credentials, but ``has_server_access`` answers True for an admin on + # any string, and the loop it starts outlives the check: the engine + # would bind to whatever gets created under that name later (SEC-164). + # Refused only *after* the access check, so a caller with no access + # still gets the same "No access" a real server gives them and this + # route never reveals which names exist. + if not cm.get_server(server_name): + raise HTTPException(status_code=404, detail="Server not found") + if req.trading_context: config_dict["trading_context"] = req.trading_context elif not config_dict.get("trading_context") and strategy.default_trading_context: diff --git a/condor/web/routes/archived.py b/condor/web/routes/archived.py index d1df8566c..d65539fa0 100644 --- a/condor/web/routes/archived.py +++ b/condor/web/routes/archived.py @@ -7,9 +7,8 @@ from fastapi import APIRouter, Depends, HTTPException, Query -from config_manager import get_config_manager from condor.fetchers.executors import normalize_executor_side -from condor.web.auth import get_current_user +from condor.web.auth import require_server_access from condor.web.models import ( ArchivedBotPerformance, ArchivedBotSummary, @@ -18,6 +17,7 @@ PnlPoint, WebUser, ) +from config_manager import get_config_manager logger = logging.getLogger(__name__) @@ -66,6 +66,7 @@ def _parse_json_field(val: Any) -> dict: return val if isinstance(val, str) and val.strip().startswith("{"): import json + try: return json.loads(val) except Exception: @@ -105,10 +106,7 @@ def _normalize_executors(raw: list[dict]) -> list[NormalizedExecutor]: ) # Resolve trading pair - trading_pair = str( - ex.get("trading_pair", "") - or config.get("trading_pair", "") - ) + trading_pair = str(ex.get("trading_pair", "") or config.get("trading_pair", "")) # Resolve prices entry_price = float( @@ -129,26 +127,32 @@ def _normalize_executors(raw: list[dict]) -> list[NormalizedExecutor]: volume = float(ex.get("filled_amount_quote", 0) or ex.get("volume", 0) or 0) net_pnl_pct = float(ex.get("net_pnl_pct", 0) or 0) - result.append(NormalizedExecutor( - id=str(ex.get("id", "") or ex.get("executor_id", "")), - type=str(ex.get("type", "") or ex.get("executor_type", "") or config.get("type", "position")), - connector=connector, - trading_pair=trading_pair, - side=normalize_executor_side(side_raw), - status=str(ex.get("status", "") or "closed"), - close_type=str(ex.get("close_type", "") or ""), - pnl=pnl, - volume=volume, - timestamp=_to_epoch_seconds(ex.get("timestamp")), - close_timestamp=_to_epoch_seconds(ex.get("close_timestamp")), - entry_price=entry_price, - current_price=close_price, - cum_fees_quote=fees, - net_pnl_pct=net_pnl_pct, - controller_id=str(ex.get("controller_id", "")), - custom_info=custom_info, - config=config, - )) + result.append( + NormalizedExecutor( + id=str(ex.get("id", "") or ex.get("executor_id", "")), + type=str( + ex.get("type", "") + or ex.get("executor_type", "") + or config.get("type", "position") + ), + connector=connector, + trading_pair=trading_pair, + side=normalize_executor_side(side_raw), + status=str(ex.get("status", "") or "closed"), + close_type=str(ex.get("close_type", "") or ""), + pnl=pnl, + volume=volume, + timestamp=_to_epoch_seconds(ex.get("timestamp")), + close_timestamp=_to_epoch_seconds(ex.get("close_timestamp")), + entry_price=entry_price, + current_price=close_price, + cum_fees_quote=fees, + net_pnl_pct=net_pnl_pct, + controller_id=str(ex.get("controller_id", "")), + custom_info=custom_info, + config=config, + ) + ) return result @@ -237,7 +241,9 @@ async def _fetch_and_cache_performance( _executors_cache[cache_key] = executors # Derive primary connector and trading pair - primary_connector, primary_trading_pair = _derive_primary_pair(executors, exchanges, trading_pairs) + primary_connector, primary_trading_pair = _derive_primary_pair( + executors, exchanges, trading_pairs + ) # Calculate PnL from trades from condor.archived_pnl import calculate_pnl_from_trades @@ -303,10 +309,8 @@ async def _fetch_and_cache_performance( @router.get("/servers/{name}/archived") -async def list_archived_bots(name: str, user: WebUser = Depends(get_current_user)): +async def list_archived_bots(name: str, user: WebUser = Depends(require_server_access)): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) @@ -342,16 +346,18 @@ async def list_archived_bots(name: str, user: WebUser = Depends(get_current_user return {"bots": bots} -@router.get("/servers/{name}/archived/performance", response_model=ArchivedBotPerformance) +@router.get( + "/servers/{name}/archived/performance", response_model=ArchivedBotPerformance +) async def get_archived_performance( name: str, db_path: str = Query(..., description="Database path"), - include_executors: bool = Query(False, description="Include full executor list in response"), - user: WebUser = Depends(get_current_user), + include_executors: bool = Query( + False, description="Include full executor list in response" + ), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) @@ -370,11 +376,9 @@ async def get_archived_executors( db_path: str = Query(..., description="Database path"), offset: int = Query(0, ge=0), limit: int = Query(50, ge=1, le=200), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") cache_key = (name, db_path) diff --git a/condor/web/routes/backtesting.py b/condor/web/routes/backtesting.py index eace66f7a..ff35163f6 100644 --- a/condor/web/routes/backtesting.py +++ b/condor/web/routes/backtesting.py @@ -6,8 +6,8 @@ from pydantic import BaseModel from condor.backtest_store import get_backtest_store -from condor.backtesting import coerce_controller_config -from condor.web.auth import get_current_user +from condor.backtesting import coerce_controller_config, normalize_backtest_task +from condor.web.auth import require_server_access from condor.web.models import WebUser from config_manager import get_config_manager @@ -28,11 +28,9 @@ class SubmitBacktestRequest(BaseModel): async def submit_backtest_task( name: str, body: SubmitBacktestRequest, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) @@ -56,11 +54,9 @@ async def submit_backtest_task( @router.get("/servers/{name}/backtesting/tasks") async def list_backtest_tasks( name: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") store = get_backtest_store() @@ -96,6 +92,7 @@ async def list_backtest_tasks( tid = task.get("task_id", "") if store.get_result(tid): task["saved"] = True + normalize_backtest_task(task) return live_tasks @@ -104,11 +101,9 @@ async def list_backtest_tasks( async def get_backtest_task( name: str, task_id: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") store = get_backtest_store() @@ -122,14 +117,14 @@ async def get_backtest_task( store.save_result(name, task_id, result) result["saved"] = True - return result + return normalize_backtest_task(result) except Exception: pass # Fallback to saved saved = store.get_result(task_id) if saved: - return {**saved, "saved": True} + return normalize_backtest_task({**saved, "saved": True}) raise HTTPException(status_code=404, detail="Task not found") @@ -138,11 +133,9 @@ async def get_backtest_task( async def delete_backtest_task( name: str, task_id: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") store = get_backtest_store() store.delete_result(task_id) @@ -161,25 +154,19 @@ async def delete_backtest_task( @router.get("/servers/{name}/backtesting/saved") async def list_saved_results( name: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): - cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") store = get_backtest_store() - return store.list_results(name) + return [normalize_backtest_task(entry) for entry in store.list_results(name)] @router.delete("/servers/{name}/backtesting/saved/{task_id}") async def delete_saved_result( name: str, task_id: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): - cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") store = get_backtest_store() deleted = store.delete_result(task_id) diff --git a/condor/web/routes/bots.py b/condor/web/routes/bots.py index e113526df..996f64b52 100644 --- a/condor/web/routes/bots.py +++ b/condor/web/routes/bots.py @@ -9,7 +9,7 @@ from fastapi import APIRouter, Depends, HTTPException from condor.fetchers.bots import build_bots_page, extract_bots_list -from condor.web.auth import get_current_user +from condor.web.auth import require_server_access from condor.web.models import ( AvailableControllersResponse, BotDetailResponse, @@ -22,6 +22,7 @@ DeployBotRequest, WebUser, ) +from condor.web.routes._errors import upstream_error from config_manager import get_config_manager from handlers.bots._shared import clean_config_for_save @@ -231,10 +232,8 @@ def _collect_bot_runs(result: Any, runs: dict[str, str]) -> None: @router.get("/servers/{name}/bots", response_model=BotsPageResponse) -async def list_bots(name: str, user: WebUser = Depends(get_current_user)): +async def list_bots(name: str, user: WebUser = Depends(require_server_access)): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") from condor.server_data_service import ServerDataType, get_server_data_service @@ -393,10 +392,10 @@ async def _with_timeout(coro, label: str, default: Any) -> Any: @router.get("/servers/{name}/bots/{bot_id}") -async def get_bot(name: str, bot_id: str, user: WebUser = Depends(get_current_user)): +async def get_bot( + name: str, bot_id: str, user: WebUser = Depends(require_server_access) +): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") import asyncio @@ -405,7 +404,8 @@ async def get_bot(name: str, bot_id: str, user: WebUser = Depends(get_current_us try: result = await client.bot_orchestration.get_bot_status(bot_id) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception("Failed to fetch status for bot '%s' on '%s'", bot_id, name) + raise upstream_error("Failed to fetch bot status", e) if not isinstance(result, dict): raise HTTPException(status_code=404, detail="Bot not found") @@ -446,10 +446,10 @@ async def get_bot(name: str, bot_id: str, user: WebUser = Depends(get_current_us "/servers/{name}/controllers/configs", response_model=AvailableControllersResponse, ) -async def list_controller_configs(name: str, user: WebUser = Depends(get_current_user)): +async def list_controller_configs( + name: str, user: WebUser = Depends(require_server_access) +): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") import asyncio @@ -501,17 +501,18 @@ async def _get_configs(): response_model=ControllerConfigDetail, ) async def get_controller_config( - name: str, config_id: str, user: WebUser = Depends(get_current_user) + name: str, config_id: str, user: WebUser = Depends(require_server_access) ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) try: result = await client.controllers.get_controller_config(config_id) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to fetch controller config '%s' from '%s'", config_id, name + ) + raise upstream_error("Failed to fetch controller config", e) if not isinstance(result, dict): raise HTTPException(status_code=404, detail="Config not found") @@ -529,7 +530,7 @@ async def update_controller_config( name: str, config_id: str, body: dict[str, Any], - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Update a saved controller config's parameters. @@ -538,8 +539,6 @@ async def update_controller_config( - { ... } (raw dict) — existing behavior preserved """ cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) @@ -584,7 +583,10 @@ async def update_controller_config( except HTTPException: raise except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to update controller config '%s' on '%s'", config_id, name + ) + raise upstream_error("Failed to save controller config", e) return {"updated": True, "config_id": config_id, "result": result} @@ -597,12 +599,10 @@ async def get_controller_source( name: str, controller_type: str, controller_name: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Fetch the Python source of a controller.""" cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) try: @@ -610,7 +610,13 @@ async def get_controller_source( controller_type, controller_name ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to fetch controller source '%s/%s' from '%s'", + controller_type, + controller_name, + name, + ) + raise upstream_error("Failed to fetch controller source", e) if isinstance(result, str): source = result @@ -639,12 +645,10 @@ async def update_controller_source( controller_type: str, controller_name: str, body: dict[str, Any], - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Update the Python source of a controller.""" cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") source = body.get("source") if not source or not isinstance(source, str): @@ -656,7 +660,13 @@ async def update_controller_source( controller_type, controller_name, {"content": source} ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to update controller source '%s/%s' on '%s'", + controller_type, + controller_name, + name, + ) + raise upstream_error("Failed to save controller source", e) return {"updated": True, "result": result} @@ -668,12 +678,10 @@ async def get_controller_config_template( name: str, controller_type: str, controller_name: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Fetch the config template/schema for a controller.""" cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) try: @@ -681,7 +689,13 @@ async def get_controller_config_template( controller_type, controller_name ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to fetch config template for '%s/%s' from '%s'", + controller_type, + controller_name, + name, + ) + raise upstream_error("Failed to fetch controller config template", e) if not result: raise HTTPException(status_code=404, detail="Template not found") @@ -696,12 +710,10 @@ async def get_controller_config_template( async def create_controller_config( name: str, body: dict[str, Any], - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Create or update a controller config.""" cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") config_id = body.get("id") if not config_id: @@ -730,7 +742,10 @@ async def create_controller_config( config_id, clean_body ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to create controller config '%s' on '%s'", config_id, name + ) + raise upstream_error("Failed to save controller config", e) return {"created": True, "config_id": config_id, "result": result} @@ -739,18 +754,19 @@ async def create_controller_config( async def delete_controller_config( name: str, config_id: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Delete a saved controller config.""" cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) try: result = await client.controllers.delete_controller_config(config_id) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to delete controller config '%s' from '%s'", config_id, name + ) + raise upstream_error("Failed to delete controller config", e) return {"deleted": True, "config_id": config_id, "result": result} @@ -760,12 +776,10 @@ async def delete_controller( name: str, controller_type: str, controller_name: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Delete a controller.""" cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) try: @@ -773,7 +787,13 @@ async def delete_controller( controller_type, controller_name ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to delete controller '%s/%s' from '%s'", + controller_type, + controller_name, + name, + ) + raise upstream_error("Failed to delete controller", e) return { "deleted": True, @@ -785,11 +805,9 @@ async def delete_controller( @router.post("/servers/{name}/bots/deploy") async def deploy_bot_endpoint( - name: str, body: DeployBotRequest, user: WebUser = Depends(get_current_user) + name: str, body: DeployBotRequest, user: WebUser = Depends(require_server_access) ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) @@ -806,18 +824,17 @@ async def deploy_bot_endpoint( max_controller_drawdown_quote=body.max_controller_drawdown_quote, ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception("Failed to deploy bot '%s' on '%s'", body.bot_name, name) + raise upstream_error("Failed to deploy bot", e) return result @router.post("/servers/{name}/bots/{bot_name}/stop") async def stop_bot_endpoint( - name: str, bot_name: str, user: WebUser = Depends(get_current_user) + name: str, bot_name: str, user: WebUser = Depends(require_server_access) ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") # Mark as stopping immediately so UI reflects it mark_bot_stopping(name, bot_name) @@ -834,7 +851,8 @@ async def stop_bot_endpoint( ) except Exception as e: clear_bot_stopping(name, bot_name) - raise HTTPException(status_code=502, detail=str(e)) + logger.exception("Failed to stop bot '%s' on '%s'", bot_name, name) + raise upstream_error("Failed to stop bot", e) return result @@ -844,11 +862,9 @@ async def stop_controllers_endpoint( name: str, bot_name: str, body: ControllerActionRequest, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") # Mark controllers as stopping immediately mark_controllers_stopping(name, bot_name, body.controller_names) @@ -865,7 +881,10 @@ async def stop_controllers_endpoint( controller_names=body.controller_names, ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to stop controllers on bot '%s' of '%s'", bot_name, name + ) + raise upstream_error("Failed to stop controllers", e) return result @@ -875,11 +894,9 @@ async def start_controllers_endpoint( name: str, bot_name: str, body: ControllerActionRequest, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) @@ -893,7 +910,10 @@ async def start_controllers_endpoint( controller_names=body.controller_names, ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to start controllers on bot '%s' of '%s'", bot_name, name + ) + raise upstream_error("Failed to start controllers", e) return result @@ -904,12 +924,10 @@ async def update_bot_controller_config_endpoint( bot_name: str, config_id: str, body: dict[str, Any], - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Update a controller config inside a running bot in real-time.""" cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) @@ -934,7 +952,13 @@ async def update_bot_controller_config_endpoint( except HTTPException: raise except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to update controller config '%s' on bot '%s' of '%s'", + config_id, + bot_name, + name, + ) + raise upstream_error("Failed to save controller config", e) return { "updated": True, diff --git a/condor/web/routes/code.py b/condor/web/routes/code.py new file mode 100644 index 000000000..7a9e50aab --- /dev/null +++ b/condor/web/routes/code.py @@ -0,0 +1,99 @@ +"""Ad-hoc code execution — the crossing into the main process (FEAT-047). + +The MCP subprocess cannot run a snippet itself: the cached API clients, the +report index's single writer and the real bot all live here. So ``run_code`` +POSTs its snippet to this route, exactly as ``manage_routines(action="run")`` +already delegates a routine run — same destination, same context, one pattern +for the next reader of ``tools/code.py`` beside ``tools/routines.py``. + +There are no GET endpoints on purpose: history is read straight off disk by the +subprocess, which shares the filesystem, just as routine CRUD is. + +A snippet runs unsandboxed in this process, so reaching this route *is* reaching +everything the bot can reach — config.yml, the web JWT secret, os.environ. The +server ACL below only picks which client is pre-resolved; it cannot constrain a +body. The real gate is therefore ``_may_run_code`` (SEC-151). +""" + +from __future__ import annotations + +import logging + +from fastapi import APIRouter, Depends, HTTPException +from pydantic import BaseModel + +from condor.code_runner import DEFAULT_TIMEOUT, MAX_TIMEOUT, execute_code +from condor.web.auth import check_server_access, get_current_user +from condor.web.models import WebUser +from config_manager import CODE_RUN_PREFERENCE, get_config_manager + +log = logging.getLogger(__name__) +router = APIRouter(prefix="/code", tags=["code"]) + +# The preference an admin sets on a user record to hand them the code runner. +# The spelling lives in config_manager beside the setter an admin drives it +# from (ARCH-177); re-exported here because this is where it is enforced. +RUN_CODE_PREFERENCE = CODE_RUN_PREFERENCE + + +def _may_run_code(user_id: int) -> bool: + """Who is allowed to execute a snippet in the bot's own process. + + Being approved is not enough. ``execute_code`` compiles with full builtins + and no sandbox, so any snippet can read every server's credentials, the web + JWT secret and ``os.environ`` no matter which servers its caller was granted + — running code here is admin-equivalent by construction, and gating it on + ``UserRole.USER`` silently voided the whole role model (SEC-151). + + Admins therefore pass, and anyone else only with the explicit + ``code_run`` preference an admin sets on their record. That opt-in is what + keeps the door open for a non-admin seat whose MCP ``run_code`` tool is a + legitimate part of their workflow: it is a grant, not a default. + """ + cm = get_config_manager() + return cm.is_admin(user_id) or bool( + cm.get_user_preference(user_id, RUN_CODE_PREFERENCE, False) + ) + + +class RunCodeRequest(BaseModel): + code: str + # The server the snippet's client resolves against. Empty is legal: a + # snippet that only needs pandas must not be refused for lack of a server. + server_name: str = "" + label: str = "" + timeout: float = DEFAULT_TIMEOUT + # Which assistant produced this run's reports, same field and same meaning + # as RunRequestV2.attribute_to. + attribute_to: str = "" + session_key: str = "" + + +@router.post("/run") +async def run_code( + body: RunCodeRequest, + user: WebUser = Depends(get_current_user), +): + """Execute a Python snippet in this process and return its recorded run.""" + if not _may_run_code(user.id): + # Before the 400 and before any store write: a refused caller must not + # learn anything, and must leave no CodeRun behind. + log.warning("Denied /code/run to user %s (not admin, no grant)", user.id) + raise HTTPException(status_code=403, detail="Not allowed to run code") + if not body.code.strip(): + raise HTTPException(400, "code is required") + if body.server_name: + check_server_access(user.id, body.server_name) + + return await execute_code( + body.code, + server_name=body.server_name, + agent=body.attribute_to, + label=body.label, + timeout=max(1.0, min(float(body.timeout or DEFAULT_TIMEOUT), MAX_TIMEOUT)), + # The caller's own id, never a value from the body: it is what resolves + # the snippet's client, so it must be the authenticated identity — the + # same one RoutineStore.execute runs a routine under. + chat_id=user.id, + session_key=body.session_key, + ) diff --git a/condor/web/routes/controller_performance.py b/condor/web/routes/controller_performance.py index 2150a940d..7dd86a517 100644 --- a/condor/web/routes/controller_performance.py +++ b/condor/web/routes/controller_performance.py @@ -3,11 +3,11 @@ import logging from typing import Optional -from fastapi import APIRouter, Depends, HTTPException, Query +from fastapi import APIRouter, Depends, Query from condor.fetchers.bot_performance import extract_snapshots as _extract_snapshots from condor.fetchers.bot_performance import fetch_all_bot_performance -from condor.web.auth import get_current_user +from condor.web.auth import require_server_access from condor.web.models import ( BotRunInfo, BotRunsResponse, @@ -16,6 +16,7 @@ ControllerPerformanceSnapshot, WebUser, ) +from condor.web.routes._errors import upstream_error from config_manager import get_config_manager logger = logging.getLogger(__name__) @@ -100,14 +101,12 @@ async def get_bot_runs( deployment_status: Optional[str] = Query(None), limit: int = Query(100, ge=1, le=500), offset: int = Query(0, ge=0), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Get bot runs with optional filtering.""" import asyncio cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) @@ -136,8 +135,8 @@ async def _fetch_perf() -> dict[str, dict]: try: result, perf_by_bot = await asyncio.gather(_fetch_runs(), _fetch_perf()) except Exception as e: - logger.warning("Failed to fetch bot runs from '%s': %s", name, e) - raise HTTPException(status_code=502, detail=str(e)) + logger.exception("Failed to fetch bot runs from '%s'", name) + raise upstream_error("Failed to fetch bot runs", e) runs_list = _extract_runs_list(result) @@ -153,20 +152,18 @@ async def _fetch_perf() -> dict[str, dict]: async def delete_bot_run( name: str, bot_run_id: int, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Delete an archived bot run by its numeric ID.""" cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) try: result = await client.bot_orchestration.delete_bot_run(bot_run_id) except Exception as e: - logger.warning("Failed to delete bot run %d from '%s': %s", bot_run_id, name, e) - raise HTTPException(status_code=502, detail=str(e)) + logger.exception("Failed to delete bot run %d from '%s'", bot_run_id, name) + raise upstream_error("Failed to delete bot run", e) return {"deleted": True, "bot_run_id": bot_run_id, "result": result} @@ -197,12 +194,10 @@ def _extract_runs_list(result) -> list[dict]: async def get_latest_controller_performance( name: str, bot_name: Optional[str] = Query(None), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Get the most recent performance snapshot for each bot/controller.""" cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) @@ -242,12 +237,10 @@ async def get_controller_performance_history( interval: str = Query("5m"), limit: Optional[int] = Query(None, ge=1, le=5000), cursor: Optional[str] = Query(None), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Get historical controller performance with pagination and interval sampling.""" cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) diff --git a/condor/web/routes/dex.py b/condor/web/routes/dex.py new file mode 100644 index 000000000..3c5781dc8 --- /dev/null +++ b/condor/web/routes/dex.py @@ -0,0 +1,584 @@ +"""Pool discovery for the DEX page. + +The DEX page is pool-first: `SOL-USDC` exists in dozens of pools across Meteora, +Orca, Raydium and Uniswap with different fee tiers, bin steps, TVL and APR, and +which one you are in *is* the decision. So this router browses pools rather than +pairs, from the same three sources Telegram's ``/lp`` flow offers. + +Everything upstream lives in ``condor.pool_data`` — the single module that +talks to GeckoTerminal — so these handlers are auth, validation and shape, with no +fetching or normalization of their own. +""" + +from __future__ import annotations + +import logging +import re + +from fastapi import APIRouter, Depends, HTTPException, Query +from pydantic import BaseModel, Field + +from condor import dex_candles +from condor.web.auth import require_server_access +from condor.web.models import WebUser +from config_manager import get_config_manager + +logger = logging.getLogger(__name__) + +router = APIRouter(tags=["dex"]) + +# A token or pool address: an EVM 0x-address or a base58 Solana pubkey. Both reach +# GeckoTerminal as a URL path segment, so they are validated before they get there +# — the same guard ``/market/candles`` and ``/market/token-symbol`` already apply. +_ADDRESS_RE = dex_candles.ADDRESS_RE +_POOL_ADDRESS_RE = _ADDRESS_RE + +# The chain the browser opens on, and the one it falls back to when Gateway cannot +# be reached: Solana is where Condor's CLMM connectors (Meteora, Orca) live. +DEFAULT_NETWORK = "solana-mainnet-beta" + +_TESTNET_RE = re.compile(r"(testnet|devnet|sepolia|goerli)", re.IGNORECASE) + +# Gateway names a network `chain-network`; the browser wants the part that varies. +_CHAIN_LABELS = { + "solana-mainnet-beta": "Solana", + "ethereum-mainnet": "Ethereum", + "ethereum-base": "Base", + "ethereum-robinhoodchain": "Robinhood", + "ethereum-unichain": "Unichain", + "ethereum-arbitrum": "Arbitrum", + "ethereum-avalanche": "Avalanche", + "ethereum-bsc": "BNB Chain", + "ethereum-celo": "Celo", + "ethereum-optimism": "Optimism", + "ethereum-polygon": "Polygon", +} + + +def _chain_label(network_id: str, item: dict) -> str: + if network_id in _CHAIN_LABELS: + return _CHAIN_LABELS[network_id] + # An unmapped network is still perfectly browsable; title-case its own name + # rather than showing a raw id or dropping it. + name = str(item.get("network") or network_id).replace("-", " ").strip() + return name.title() or network_id + + +@router.get("/servers/{name}/dex/pools") +async def list_pools( + name: str, + source: str = Query( + default="gecko", + description="'gecko' (a chain's trending/top/new pools, or a token's), " + "'orca' (Orca's own whirlpool API, with realized fees and yield) or " + "'gateway' (a CLMM connector's own listing, with APR and bin step)", + ), + network: str = Query( + default="solana-mainnet-beta", + description="source=gecko: the chain, as a Gateway network or gecko id", + ), + view: str = Query( + default="trending", description="source=gecko: trending | top | new | token" + ), + connector: str = Query( + default="meteora", description="source=gateway: meteora | orca | raydium | ..." + ), + query: str | None = Query( + default=None, + description="source=gecko+view=token: the token address. " + "source=gateway: free text matched against pool names.", + ), + dexes: str | None = Query( + default=None, + description="source=gecko: comma-separated GeckoTerminal dex ids to keep " + "(meteora,orca,raydium-clmm). Empty means every venue.", + ), + limit: int = Query(default=20, ge=1, le=100), + page: int = Query(default=1, ge=1, description="1-based page of `limit` rows"), + user: WebUser = Depends(require_server_access), +): + """Pools to browse, from one of the three upstreams. + + ``source`` is the discriminator because they take genuinely different arguments + — gecko searches by *token address* on a chain, Gateway and Orca search free + text within a venue — and one merged parameter set would mean documenting which + combinations are meaningless. + + An upstream failure answers ``{"pools": []}`` with a warning, like + ``/market/venues``: the browser's empty state is something the user can act on, + a 502 is not. ``source=orca`` falls back to Gateway before it comes to that. + + ``has_more`` — not a total — is what the browser's Next needs. Orca answers it + exactly, having the whole ranked set in hand; Gateway and gecko report no total, + so there a full page is the only evidence of another. + """ + cm = get_config_manager() + + from condor.pool_data import ( + list_gateway_pools, + list_gecko_pools_page, + list_orca_pools, + ) + + source = (source or "gecko").strip().lower() + + if source == "orca": + # Orca's own API, not Gateway's proxy of it: same pools, but with the + # volume, fees, price and yield the proxy nulls, and with pagination that + # is not stuck on page one. Gateway stays the fallback for when Orca is + # rate-limiting — a degraded row beats an empty table. + paged = await list_orca_pools(search=query, limit=limit, page=page) + if paged is not None: + pools, has_more = paged + return { + "pools": pools, + "source": source, + "page": page, + "has_more": has_more, + } + logger.warning("Orca API unavailable for %s; falling back to Gateway", name) + connector, source = "orca", "gateway" + + if source == "gateway": + try: + client = await cm.get_client(name) + except Exception as e: + logger.warning("Gateway pools unavailable for %s: %s", name, e) + return {"pools": [], "source": source, "page": page, "has_more": False} + pools = await list_gateway_pools( + client, connector, search=query, limit=limit, page=page + ) + # Gateway reports no total, so a full page is the only evidence of another. + return { + "pools": pools, + "source": source, + "page": page, + "has_more": len(pools) >= limit, + } + + if source != "gecko": + raise HTTPException(status_code=400, detail="Unknown source") + + view = (view or "trending").strip().lower() + token = (query or "").strip() + # A ticker would be pasted straight into a GeckoTerminal path segment. + if view == "token" and not _ADDRESS_RE.match(token): + raise HTTPException(status_code=400, detail="Invalid token address") + + before = _throttle_counter() + result = await list_gecko_pools_page( + network, + view=view, + token=token, + limit=limit, + page=page, + dexes=[d for d in (dexes or "").split(",") if d.strip()], + ) + return { + "pools": result["pools"], + "source": source, + "page": page, + "has_more": result["has_more"], + # Reported on the same response as the rows so the browser can say why a + # page came back short, without a second round-trip to find out. + "upstream": _upstream_state(_throttle_counter() > before), + } + + +@router.get("/servers/{name}/dex/dexes") +async def list_dexes( + name: str, + network: str = Query(default="solana-mainnet-beta"), + user: WebUser = Depends(require_server_access), +): + """The venues on a chain, for the pool browser's dex filter. + + Offered by the chain rather than hardcoded, so a venue GeckoTerminal starts + indexing is filterable without a deploy. Empty when the lookup fails — a filter + with no options is one the browser simply does not show. + """ + + from condor.pool_data import list_gecko_dexes + + return {"dexes": await list_gecko_dexes(network)} + + +@router.get("/servers/{name}/dex/chains") +async def list_chains( + name: str, + user: WebUser = Depends(require_server_access), +): + """The chains the pool browser can offer, from Gateway rather than a constant. + + A chain qualifies only if it is both something *this* Gateway is configured for + and something GeckoTerminal indexes — a network Gateway reaches but Gecko does + not has no pools to browse, and one Gecko indexes but Gateway cannot reach + routes to a workspace that cannot trade. Testnets are dropped for the same + reason: there is nothing to discover on them. + + Falls back to Solana alone when Gateway is unreachable, so the browser always + has at least the chain it defaults to. + """ + cm = get_config_manager() + + from condor.pool_data import NETWORK_TO_GECKO + + fallback = [{"network_id": DEFAULT_NETWORK, "chain": "solana", "label": "Solana"}] + try: + client = await cm.get_client(name) + result = await client.gateway.list_networks() + except Exception as e: + logger.warning("Gateway networks unavailable for %s: %s", name, e) + return {"chains": fallback} + + raw = result.get("networks", result) if isinstance(result, dict) else result + chains = [] + for item in raw if isinstance(raw, list) else []: + if not isinstance(item, dict): + continue + network_id = str(item.get("network_id") or "").strip() + if not network_id or network_id not in NETWORK_TO_GECKO: + continue + if _TESTNET_RE.search(network_id): + continue + chains.append( + { + "network_id": network_id, + "chain": str(item.get("chain") or ""), + "label": _chain_label(network_id, item), + } + ) + + if not chains: + return {"chains": fallback} + # Solana first — it is the default and the only chain with CLMM connectors + # behind it — then alphabetically, so the order does not shift with Gateway's. + chains.sort(key=lambda c: (c["network_id"] != DEFAULT_NETWORK, c["label"].lower())) + return {"chains": chains} + + +def _throttle_counter() -> int: + """Every refusal this process has made, as one monotonic number. + + Both refusals count: a 429 from gecko and a request this process declined to + send because the shared budget was already spent. The second one leaves no + trace in ``rate_limited`` — it raises without opening the breaker — so a + counter is the only way a caller can ask "was *my* fetch throttled?". + """ + from condor.pool_data import gecko_health + + health = gecko_health() + return int(health.get("throttled_calls") or 0) + int(health.get("count_429") or 0) + + +def _upstream_state(throttled_request: bool = False) -> dict: + """What GeckoTerminal's shared budget looks like right now. + + Every DEX view — the pool browser, a pool's stats, its chart — reads the same + per-IP budget, and every failure downstream of it degrades to an empty list. + So "no pools on this chain" and "throttled, ask again in a moment" arrive at + the browser looking identical unless the state is reported alongside the rows. + + ``throttled_request`` is the stronger signal of the two and belongs to the + call being answered: the budget can be spent without the breaker being open, + in which case ``rate_limited`` is False while the rows are still missing. + """ + from condor.pool_data import gecko_health + + health = gecko_health() + last_429 = health.get("last_429_age") + return { + "rate_limited": bool(health.get("rate_limited")), + "throttled_request": bool(throttled_request), + "retry_in": round(float(health.get("cooldown_remaining") or 0.0), 1), + "requests_last_minute": int(health.get("requests_last_minute") or 0), + "budget": int(health.get("budget") or 0), + "throttled_calls": int(health.get("throttled_calls") or 0), + "count_429": int(health.get("count_429") or 0), + "seconds_since_429": ( + round(float(last_429), 1) if last_429 is not None else None + ), + } + + +@router.get("/servers/{name}/dex/upstream") +async def get_upstream( + name: str, + user: WebUser = Depends(require_server_access), +): + """Whether GeckoTerminal is currently throttling Condor. + + Reads process-local counters only — no outbound request — so the browser can + poll it while a banner counts a cooldown down without spending the very + budget it is waiting on. Declared above ``/dex/pools/{pool_address}`` for the + same reason ``pools-by-address`` is: a path segment would be read as an + address. + """ + return _upstream_state() + + +class EnsureTokensRequest(BaseModel): + """The tokens a pool is made of, as the browser already knows them.""" + + network: str = Field(description="Gateway network id, e.g. solana-mainnet-beta") + addresses: list[str] = Field( + default_factory=list, + description="Token mints/contracts to register — a pool's base and quote", + ) + + +@router.post("/servers/{name}/dex/tokens") +async def ensure_dex_tokens( + name: str, + body: EnsureTokensRequest, + user: WebUser = Depends(require_server_access), +): + """Make sure Gateway's token list holds this pool's tokens. + + Called when a pool workspace opens rather than when an order is placed, + because the panel needs the tokens *before* the trade: an unlisted mint reads + as a zero balance, which greys out the percentage presets and sizes an LP + position against a wallet that looks empty. Idempotent and memoized, so + re-opening a pool costs nothing. + + Never fatal. A pool whose tokens could not be registered is still browsable, + chartable and (on Solana, where Gateway resolves a mint on chain) tradable — + so a failure is reported per token and left for the panel to surface, not + raised. + """ + from condor.fetchers.gateway_tokens import ensure_tokens_listed, token_addresses + from condor.pool_data import GECKO_TO_GATEWAY_NETWORK, NETWORK_TO_GECKO + + # Canonicalized rather than taken as given: a pool row can name its chain the + # gecko way (``solana``) while Gateway only answers to ``solana-mainnet-beta``. + gateway_network = GECKO_TO_GATEWAY_NETWORK.get( + NETWORK_TO_GECKO.get(body.network, "") + ) + if not gateway_network: + raise HTTPException(status_code=400, detail="Unknown network") + + addresses = token_addresses(*body.addresses) + if not addresses: + return {"tokens": {}} + + cm = get_config_manager() + client = await cm.get_client(name) + try: + verdicts = await ensure_tokens_listed(client, gateway_network, addresses) + except Exception as e: + logger.warning("ensure tokens failed on %s: %s", gateway_network, e) + verdicts = {address: "failed" for address in addresses} + return {"tokens": verdicts} + + +@router.get("/servers/{name}/dex/pools-by-address") +async def list_pools_by_address( + name: str, + addresses: str = Query(description="Comma-separated pool addresses (max 30)"), + network: str = Query(default="solana-mainnet-beta"), + user: WebUser = Depends(require_server_access), +): + """Several pools by address at once, for the favourites view. + + A favourite is stored as an address, not as a copy of the row, so its TVL and + volume have to be re-read like any other row rather than replayed from + whenever it was starred. An address that no longer resolves is absent from the + answer instead of failing the whole view — a de-indexed pool is not an error. + + Declared above ``/dex/pools/{pool_address}`` deliberately: a path segment would + otherwise be read as an address. + """ + + from condor.pool_data import fetch_pools_by_addresses + + wanted = [a.strip() for a in (addresses or "").split(",") if a.strip()][:30] + if not wanted: + return {"pools": [], "upstream": _upstream_state()} + before = _throttle_counter() + pools = await fetch_pools_by_addresses(network, wanted) + return {"pools": pools, "upstream": _upstream_state(_throttle_counter() > before)} + + +@router.get("/servers/{name}/dex/pools/{pool_address}") +async def get_pool( + name: str, + pool_address: str, + network: str = Query(default="solana-mainnet-beta"), + user: WebUser = Depends(require_server_access), +): + """One pool by address, so ``/dex/{network}/{address}`` renders from a URL alone. + + 404 when the pool is genuinely unknown, 503 when the lookup was refused by the + shared GeckoTerminal budget: the workspace has nothing to draw either way, but + only one of the two is worth retrying, and the user cannot tell them apart + unless the route does. + """ + + if not _POOL_ADDRESS_RE.match(pool_address): + raise HTTPException(status_code=400, detail="Invalid pool_address") + + from condor.pool_data import fetch_pool_by_address + + before = _throttle_counter() + pool = await fetch_pool_by_address(network, pool_address) + if not pool: + # A throttled lookup is a different answer from an unknown pool, and the + # difference is the whole point: one is worth retrying in a moment, the + # other never resolves. 503 rather than 404 so the browser can say so and + # offer a retry instead of sending the user back to the browser. + state = _upstream_state(_throttle_counter() > before) + if state["throttled_request"] or state["rate_limited"]: + wait = state["retry_in"] or 20 + raise HTTPException( + status_code=503, + detail=( + "GeckoTerminal is rate limiting Condor, so this pool could " + f"not be read. Try again in about {wait:.0f}s." + ), + ) + raise HTTPException(status_code=404, detail="Pool not found") + return pool + + +def _no_bins(reason: str) -> dict: + """The unavailable state, with a 200. + + A pool Condor cannot draw bins for is something to render — a one-line + reason where the depth column would be — not a failure the user can act on. + Upstream failures degrade the same way, for the same reason. + """ + return { + "bins": [], + "active_price": None, + "bin_step": None, + "available": False, + "reason": reason, + } + + +def _as_float(value) -> float | None: + try: + number = float(value) + except (TypeError, ValueError): + return None + return number if number == number else None # NaN is not a price + + +@router.get("/servers/{name}/dex/pools/{pool_address}/bins") +async def get_pool_bins( + name: str, + pool_address: str, + network: str = Query(default="solana-mainnet-beta"), + connector: str = Query( + default="meteora", + description="The pool's Gateway CLMM connector, or the GeckoTerminal " + "dex_id it is derived from (meteora-dlmm → meteora).", + ), + user: WebUser = Depends(require_server_access), +): + """The pool's liquidity bins, for the depth column beside its chart. + + ``can_fetch_liquidity`` is the gate and is asked first — it is the same + predicate that decorates every pool row with ``has_bins``, so the workspace + already knows whether to call. It answers a *different* question from + ``lp_provider_for_dex``: a pool can be LP-able without Condor being able to + draw its bins, and the two must not be read as proxies for each other. + + The ``get_pool_info`` → ``get_pools`` fallback for pools outside Gateway's + DLMM list stays where it already is, in ``fetch_liquidity_bins``; this + handler injects a client and shapes the answer. + """ + cm = get_config_manager() + + if not _POOL_ADDRESS_RE.match(pool_address): + raise HTTPException(status_code=400, detail="Invalid pool_address") + + from condor.pool_data import ( + can_fetch_liquidity, + fetch_liquidity_bins, + get_connector_for_dex, + ) + + # Gated on the *dex id* as given, which is exactly what decorates each pool + # row with ``has_bins`` — so the route and the browser never disagree about + # which pools have a depth column. A Raydium AMM v4 pool is refused here for + # the same reason it is refused there: only ``raydium`` (CLMM) is in the set. + named = (connector or "").strip().lower() + if not named or not can_fetch_liquidity(named, network): + return _no_bins( + f"Condor reads liquidity bins from Gateway CLMM, which does not cover " + f"{named or 'this venue'} on {network}." + ) + # The Gateway connector the gated dex id names (``meteora`` → ``meteora``). + resolved = get_connector_for_dex(named) or named + + try: + client = await cm.get_client(name) + except Exception as e: + logger.warning("Gateway client unavailable for %s: %s", name, e) + client = None + if client is None: + return _no_bins("The API server is not reachable, so bins cannot be read.") + + try: + bins, pool_info, error = await fetch_liquidity_bins( + pool_address=pool_address, + connector=resolved, + network=network, + client=client, + ) + except Exception as e: # fetch_liquidity_bins already swallows its own + logger.warning("Liquidity bins failed for %s: %s", pool_address[:12], e) + return _no_bins("Liquidity bins could not be read from Gateway.") + + if error or not bins: + return _no_bins(error or "Gateway reports no liquidity bins for this pool.") + + pool_info = pool_info or {} + base_usd = _as_float(pool_info.get("base_token_price_usd")) + quote_usd = _as_float(pool_info.get("quote_token_price_usd")) + + rows = [] + for raw in bins: + if not isinstance(raw, dict): + continue + price = _as_float(raw.get("price")) + if price is None: + continue + base_amount = _as_float(raw.get("base_token_amount")) or 0.0 + quote_amount = _as_float(raw.get("quote_token_amount")) or 0.0 + # Only when both sides can be priced: a half-priced bin would size its + # bar against a different unit than its neighbours. + liquidity_usd = ( + base_amount * base_usd + quote_amount * quote_usd + if base_usd is not None and quote_usd is not None + else None + ) + rows.append( + { + "price": price, + "base_amount": base_amount, + "quote_amount": quote_amount, + "liquidity_usd": liquidity_usd, + } + ) + + if not rows: + return _no_bins("Gateway reports no liquidity bins for this pool.") + + active_price = _as_float(pool_info.get("price")) + if active_price is None: + active_price = _as_float(pool_info.get("current_price")) + bin_step = pool_info.get("bin_step") + try: + bin_step = int(bin_step) if bin_step is not None else None + except (TypeError, ValueError): + bin_step = None + + return { + "bins": rows, + "active_price": active_price, + "bin_step": bin_step, + "available": True, + "reason": None, + } diff --git a/condor/web/routes/executors.py b/condor/web/routes/executors.py index 9038f461a..26e2098a0 100644 --- a/condor/web/routes/executors.py +++ b/condor/web/routes/executors.py @@ -8,22 +8,18 @@ logger = logging.getLogger(__name__) -from config_manager import get_config_manager -from condor.web.auth import get_current_user +from condor.fetchers.executors import EXECUTORS_POLL_MAX, MAX_EXECUTORS_FETCH +from condor.fetchers.executors import extract_executors_list as _extract_executors_list +from condor.fetchers.executors import fetch_all_executors, summarize_executors_by_quote +from condor.web.auth import require_server_access from condor.web.models import ( CreateExecutorRequest, ExecutorInfo, ExecutorPeriodSummary, WebUser, ) -from condor.fetchers.executors import ( - describe_executor_error, - fetch_all_executors, - extract_executors_list as _extract_executors_list, - summarize_executors_by_quote, - EXECUTORS_POLL_MAX, - MAX_EXECUTORS_FETCH, -) +from condor.web.routes._errors import upstream_error +from config_manager import get_config_manager router = APIRouter(tags=["executors"]) @@ -46,24 +42,6 @@ _summary_cache: dict[tuple[str, str], tuple[float, ExecutorPeriodSummary]] = {} -def _executor_error(action: str, exc: Exception) -> HTTPException: - """Map an executor call failure to an HTTPException that leaks nothing. - - An upstream 4xx is the caller's own bad request and stays a 400; anything - else — an upstream 5xx, a timeout, a refused connection — is the gateway - failing, so 502. The detail carries the API's message but never the raw - exception, whose string embeds the backend URL and port. - - Reads go through this too, not only mutations: the listing endpoints are the - dashboard's hottest path, so any backend blip is the most reachable way for - that address to reach a browser. Callers log the exception before raising — - the address belongs in the server log, not in the response. - """ - status, message = describe_executor_error(exc) - code = 400 if status is not None and 400 <= status < 500 else 502 - return HTTPException(status_code=code, detail=f"{action}: {message}") - - @router.get("/servers/{name}/executors", response_model=list[ExecutorInfo]) async def list_executors( name: str, @@ -71,12 +49,15 @@ async def list_executors( trading_pair: str = Query(default="", description="Filter by trading pair"), status: str = Query(default="", description="Filter by status"), controller_id: str = Query(default="", description="Filter by controller id"), - limit: int = Query(default=0, ge=0, le=MAX_EXECUTORS_FETCH, description="Max executors to return (0 = default SDS cache)"), - user: WebUser = Depends(get_current_user), + limit: int = Query( + default=0, + ge=0, + le=MAX_EXECUTORS_FETCH, + description="Max executors to return (0 = default SDS cache)", + ), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") from condor.server_data_service import ServerDataType, get_server_data_service @@ -102,13 +83,15 @@ async def list_executors( result = executors_list except Exception as e: logger.exception("Failed to fetch executors for server %s", name) - raise _executor_error("Failed to fetch executors", e) + raise upstream_error("Failed to fetch executors", e) else: try: - result = await get_server_data_service().get_or_fetch(name, ServerDataType.EXECUTORS) + result = await get_server_data_service().get_or_fetch( + name, ServerDataType.EXECUTORS + ) except Exception as e: logger.exception("Failed to fetch executors for server %s", name) - raise _executor_error("Failed to fetch executors", e) + raise upstream_error("Failed to fetch executors", e) if result is None: raise HTTPException(status_code=502, detail="Failed to fetch executors") @@ -145,7 +128,7 @@ async def list_executors_page( trading_pair: str = Query(default=""), status: str = Query(default=""), controller_id: str = Query(default=""), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Fetch a single page of executors with a next_cursor for progressive loading. @@ -157,8 +140,6 @@ async def list_executors_page( from the API instead, so the stream continues past the poll's page budget. """ cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") has_filters = bool(executor_type or trading_pair or status or controller_id) @@ -192,7 +173,7 @@ async def list_executors_page( rows = await fetch_all_executors(client, max_items=offset + limit + 1) except Exception as e: logger.exception("Failed to page executors for server %s", name) - raise _executor_error("Failed to fetch executors", e) + raise upstream_error("Failed to fetch executors", e) return _offset_page(rows, offset, limit) # Cold cache on the first page: fall through to opaque API cursors, # which page the rest of the scroll in one request each. @@ -214,7 +195,7 @@ async def list_executors_page( result = await client.executors.search_executors(**kwargs) except Exception as e: logger.exception("Failed to page executors for server %s", name) - raise _executor_error("Failed to fetch executors", e) + raise upstream_error("Failed to fetch executors", e) page = _extract_executors_list(result) next_cursor = None @@ -278,7 +259,7 @@ async def _usd_summary( async def executors_summary( name: str, period: str = Query(default="1D", description="Window: 1D, 1W or 1M"), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """PnL, volume and executor count over a period, across the whole history. @@ -293,8 +274,6 @@ async def executors_summary( at its one request per tick. """ cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") period = period.upper() window = _PERIOD_SECONDS.get(period) @@ -315,7 +294,7 @@ async def executors_summary( executors = await fetch_all_executors(client) except Exception as e: logger.exception("Failed to summarize executors for server %s", name) - raise _executor_error("Failed to fetch executors", e) + raise upstream_error("Failed to fetch executors", e) summary = await _usd_summary( name, period, summarize_executors_by_quote(executors, now - window) @@ -324,27 +303,61 @@ async def executors_summary( return summary +async def _ensure_dex_tokens_listed(client, config: dict) -> None: + """Register a DEX executor's tokens with Gateway before it is created. + + The pool workspace already does this when it opens, so for the web UI this is + a no-op resolved from an in-process memo. It exists for the callers that never + open one — agents, Telegram, MCP — because an unlisted mint reads as a zero + balance in Gateway, which is enough to have an order rejected for funds the + wallet is holding. + + Best effort by design: a token that cannot be registered is not a reason to + refuse an order Gateway may well execute anyway. + """ + connector = str(config.get("connector_name") or "") + if not connector: + return + try: + from condor.fetchers.gateway_tokens import ( + ensure_tokens_listed, + token_addresses, + ) + from condor.pool_data import GECKO_TO_GATEWAY_NETWORK, NETWORK_TO_GECKO + + # A CEX connector ("binance") maps to no chain and drops out here; only a + # Gateway network id ("solana-mainnet-beta") reaches the token list. + network = GECKO_TO_GATEWAY_NETWORK.get(NETWORK_TO_GECKO.get(connector, "")) + if not network: + return + addresses = token_addresses(config.get("trading_pair")) + if addresses: + await ensure_tokens_listed(client, network, addresses) + except Exception as e: + logger.warning("could not ensure tokens for %s: %s", connector, e) + + @router.post("/servers/{name}/executors") async def create_executor_endpoint( name: str, body: CreateExecutorRequest, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) # Inject executor type into config config = {**body.config, "type": body.executor_type} + await _ensure_dex_tokens_listed(client, config) + from condor.fetchers.executors import create_executor try: result = await create_executor(client, config, account_name=body.account_name) except Exception as e: - raise _executor_error("Failed to create executor", e) + raise upstream_error("Failed to create executor", e) executor_id = "" if isinstance(result, dict): executor_id = str(result.get("executor_id") or result.get("id") or "") @@ -356,11 +369,9 @@ async def stop_executor_endpoint( name: str, executor_id: str, keep_position: bool = Query(default=False), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) @@ -369,25 +380,23 @@ async def stop_executor_endpoint( try: result = await stop_executor(client, executor_id, keep_position=keep_position) except Exception as e: - raise _executor_error("Failed to stop executor", e) + raise upstream_error("Failed to stop executor", e) return {"status": "ok", "result": result} @router.get("/servers/{name}/executors/positions") async def get_positions_held( name: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) try: result = await client.executors.get_positions_summary() except Exception as e: logger.exception("Failed to fetch held positions for server %s", name) - raise _executor_error("Failed to fetch positions", e) + raise upstream_error("Failed to fetch positions", e) # Normalize: extract positions list from various shapes if isinstance(result, dict): @@ -399,7 +408,10 @@ async def get_positions_held( else: positions = [] - return {"positions": positions, "summary": result if isinstance(result, dict) else {}} + return { + "positions": positions, + "summary": result if isinstance(result, dict) else {}, + } @router.delete("/servers/{name}/executors/positions/{connector}/{pair}") @@ -408,11 +420,9 @@ async def clear_position_held( connector: str, pair: str, controller_id: str = Query(default=""), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) try: @@ -427,5 +437,5 @@ async def clear_position_held( ) except Exception as e: logger.exception("Failed to clear held position on server %s", name) - raise _executor_error("Failed to clear position", e) + raise upstream_error("Failed to clear position", e) return {"status": "ok", "result": result} diff --git a/condor/web/routes/market.py b/condor/web/routes/market.py index 6c9058625..32d48706e 100644 --- a/condor/web/routes/market.py +++ b/condor/web/routes/market.py @@ -1,5 +1,6 @@ from __future__ import annotations +import asyncio import logging import time @@ -29,7 +30,19 @@ def _candle_cache_put(key: tuple, value: list, now: float) -> None: _candle_cache.pop(next(iter(_candle_cache))) -from condor.web.auth import get_current_user +# In-flight coalescing, keyed exactly like _candle_cache. The TTL cache only +# helps requests that arrive *after* one has finished; a dashboard rendering a +# grid of executor panels for the same pair and window misses a cold cache in +# all of them at once, and each miss used to fire its own upstream call — N +# duplicate GeckoTerminal requests against the process-wide rate budget, or N +# duplicate historical-candle calls to the API server. Same idiom as +# `_snapshot_inflight` in condor/fetchers/bot_performance.py: one task per key, +# popped by a done-callback so a failure is retried rather than remembered. +_candle_inflight: dict[tuple, tuple[asyncio.AbstractEventLoop, asyncio.Task]] = {} + + +from condor.fetchers.market_data import fetch_historical_candles +from condor.web.auth import get_current_user, require_server_access from condor.web.models import ( CandleData, MarketPriceResponse, @@ -42,6 +55,7 @@ def _candle_cache_put(key: tuple, value: list, now: float) -> None: TradingRulesResponse, WebUser, ) +from condor.web.routes._errors import upstream_error router = APIRouter(tags=["market"]) @@ -69,10 +83,7 @@ async def _fetch_dex_candles( @router.get("/servers/{name}/market/connectors") -async def get_connectors(name: str, user: WebUser = Depends(get_current_user)): - cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") +async def get_connectors(name: str, user: WebUser = Depends(require_server_access)): from condor.server_data_service import ServerDataType, get_server_data_service @@ -81,16 +92,16 @@ async def get_connectors(name: str, user: WebUser = Depends(get_current_user)): name, ServerDataType.CANDLE_CONNECTORS ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception("Failed to fetch candle connectors from '%s'", name) + raise upstream_error("Failed to fetch connectors", e) return result @router.get("/servers/{name}/market/connected-exchanges") -async def get_connected_exchanges(name: str, user: WebUser = Depends(get_current_user)): +async def get_connected_exchanges( + name: str, user: WebUser = Depends(require_server_access) +): """Get connectors that have credentials configured (accounts connected).""" - cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") from condor.server_data_service import ServerDataType, get_server_data_service @@ -99,20 +110,108 @@ async def get_connected_exchanges(name: str, user: WebUser = Depends(get_current name, ServerDataType.CONNECTORS ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception("Failed to fetch connected exchanges from '%s'", name) + raise upstream_error("Failed to fetch connectors", e) return result or [] +@router.get("/servers/{name}/market/venues") +async def get_venues(name: str, user: WebUser = Depends(require_server_access)): + """Venues the trade panel can offer, each with its independent traits. + + ``{"venues": [{"name", "hummingbot_market_data", "clmm_lp"}, ...]}``. The panel + maps traits to UI decisions itself — which tabs and strategies to render is a + product decision, not a server fact; only the facts they rest on come from here. + + Answers ``{"venues": []}`` rather than a 502 when the server cannot be described + at all, so the panel renders its empty state. The fetcher runs ``strict=True``, + so that failure is never cached; a gateway-only failure degrades inside the + fetcher and still reports the credentialed venues. + """ + + from condor.server_data_service import ServerDataType, get_server_data_service + + try: + result = await get_server_data_service().get_or_fetch( + name, ServerDataType.VENUES + ) + except Exception as e: + logger.warning("Venues unavailable for %s: %s", name, e) + return {"venues": []} + return {"venues": result or []} + + +_NO_POOL = { + "pool_address": None, + "dex_id": None, + "lp_provider": None, + "lp_supported": False, + "current_price": None, + "base_symbol": None, + "quote_symbol": None, +} + + +@router.get("/servers/{name}/market/dex-pool") +async def get_dex_pool( + name: str, + connector: str = Query( + ..., description="Gateway network, e.g. solana-mainnet-beta" + ), + trading_pair: str = Query(...), + user: WebUser = Depends(require_server_access), +): + """The pool a DEX pair trades in, and whether an LP position can be opened in it. + + Makes explicit the choice the chart already makes implicitly: candles for a DEX + pair come from whichever pool ``resolve_pool_info`` picks, and the LP panel has + to name that same pool. ``lp_provider`` is the ``dex/trading_type`` string an + ``lp_executor`` requires, and is ``None`` with ``lp_supported: false`` whenever + the deepest pool is not on one of the five CLMM venues — a router pool, a plain + AMM, or Uniswap v4. That is a 200, not an error: the panel's answer to it is to + ask for a pool address by hand, which is a state to render rather than a failure + to handle. + """ + + from condor.pool_data import lp_provider_for_dex, resolve_pool_info + + try: + info = await resolve_pool_info(connector, trading_pair) + except Exception as e: + logger.warning( + "Pool resolution failed connector=%s pair=%s: %s", + connector, + trading_pair, + e, + ) + return dict(_NO_POOL) + + address = str((info or {}).get("address") or "") + # A resolved address that does not look like one would be handed straight back + # to /market/candles and the executor config, so it is refused here instead. + if not info or not _POOL_ADDRESS_RE.match(address): + return dict(_NO_POOL) + + dex_id = str(info.get("dex_id") or "") + provider = lp_provider_for_dex(dex_id, connector) + return { + "pool_address": address, + "dex_id": dex_id or None, + "lp_provider": provider, + "lp_supported": provider is not None, + "current_price": info.get("current_price"), + "base_symbol": info.get("base_symbol"), + "quote_symbol": info.get("quote_symbol"), + } + + @router.get("/servers/{name}/market/prices", response_model=MarketPriceResponse) async def get_price( name: str, connector: str = Query(...), trading_pair: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): - cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") from condor.server_data_service import ServerDataType, get_server_data_service @@ -124,7 +223,10 @@ async def get_price( trading_pair=trading_pair, ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to fetch price for %s on %s of '%s'", trading_pair, connector, name + ) + raise upstream_error("Failed to fetch price", e) if result is None: raise HTTPException(status_code=502, detail="Failed to fetch price") @@ -148,12 +250,9 @@ async def get_price( async def get_rates( name: str, body: dict, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """Cross-rates resolved from Condor's cached ticker pool (replaces the rate oracle).""" - cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") trading_pairs = body.get("trading_pairs") or [] if not trading_pairs: @@ -164,7 +263,8 @@ async def get_rates( try: rates = await resolve(name, trading_pairs, connector=body.get("connector")) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception("Failed to resolve rates on '%s'", name) + raise upstream_error("Failed to fetch rates", e) return RatesResponse(rates=rates) @@ -173,11 +273,8 @@ async def get_rates( async def get_trading_rules( name: str, connector: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): - cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") from condor.fetchers._identifiers import IdentifierError, validate_identifier from condor.server_data_service import ServerDataType, get_server_data_service @@ -194,7 +291,10 @@ async def get_trading_rules( name, ServerDataType.TRADING_RULES, connector_name=connector ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to fetch trading rules for '%s' on '%s'", connector, name + ) + raise upstream_error("Failed to fetch trading rules", e) if not isinstance(result, dict): return TradingRulesResponse(connector=connector, rules=[]) @@ -220,19 +320,17 @@ async def get_trading_rules( async def get_tickers( name: str, connector: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): """24h tickers for a connector, sorted by USD volume (highest first).""" - cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") from condor.market_rates import get_connector_tickers try: result = await get_connector_tickers(name, connector) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception("Failed to fetch tickers for '%s' on '%s'", connector, name) + raise upstream_error("Failed to fetch tickers", e) if not isinstance(result, dict): return TickersResponse(connector=connector, tickers=[]) @@ -258,11 +356,9 @@ async def get_order_book( connector: str = Query(...), trading_pair: str = Query(...), depth: int = Query(default=20, ge=1, le=100), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") client = await cm.get_client(name) try: @@ -270,7 +366,13 @@ async def get_order_book( connector_name=connector, trading_pair=trading_pair ) except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) + logger.exception( + "Failed to fetch order book for %s on %s of '%s'", + trading_pair, + connector, + name, + ) + raise upstream_error("Failed to fetch order book", e) bids = [] asks = [] @@ -305,6 +407,64 @@ async def get_order_book( ) +async def _fetch_candles_upstream( + cm, + name: str, + connector: str, + trading_pair: str, + interval: str, + limit: int, + start_time: float | None, + end_time: float | None, + pool_address: str | None, +) -> list[CandleData]: + """Fetch one window of candles upstream, with no caching of its own. + + Split out of the route so concurrent requests on the same cache key can + share a single execution of it (see `_candle_inflight`). Raises exactly what + the route used to raise inline, so every waiter on a failing fetch sees the + same error and nothing is cached. + """ + # Pairs with no CandlesFactory feed never reach the client below — that path + # 502s for them. A GeckoTerminal-backed connector (a DEX network, or an + # on-chain venue like xrpl) or an explicit pool_address routes there instead. + if pool_address or dex_candles.uses_gecko_candles(connector): + return await _fetch_dex_candles( + connector, pool_address, trading_pair, interval, start_time, end_time + ) + + client = await cm.get_client(name) + try: + rows = await fetch_historical_candles( + client, + connector, + trading_pair, + interval, + # No start_time means no time range to ask for: go straight to the + # plain `limit`-sized window. + start_time=int(start_time) if start_time is not None else None, + end_time=int(end_time) if end_time else int(time.time()), + limit=limit, + fallback_on_error=True, + # CORR-168: one malformed row costs one candle, not the whole + # chart. The fetcher logs what it dropped at warning, so a payload + # bug is still diagnosable without 500ing every request for the + # pair. A real upstream failure still raises below and is wrapped + # as `upstream_error` — the two stay distinguishable. + strict=False, + ) + except Exception as e: + logger.exception( + "Failed to fetch candles for %s on %s of '%s'", + trading_pair, + connector, + name, + ) + raise upstream_error("Failed to fetch candles", e) + + return [CandleData(**row) for row in rows] + + @router.get("/servers/{name}/market/candles", response_model=list[CandleData]) async def get_candles( name: str, @@ -320,11 +480,9 @@ async def get_candles( "connector (e.g. solana-mainnet-beta) has no CandlesFactory feed, so " "candles come from GeckoTerminal by pool instead.", ), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") if pool_address and not _POOL_ADDRESS_RE.match(pool_address): raise HTTPException(status_code=400, detail="Invalid pool_address") @@ -347,100 +505,39 @@ async def get_candles( if cached and (now - cached[0]) < _CANDLE_CACHE_TTL: return cached[1] - # Pairs with no CandlesFactory feed never reach the client below — that path - # 502s for them. A GeckoTerminal-backed connector (a DEX network, or an - # on-chain venue like xrpl) or an explicit pool_address routes there instead. - if pool_address or dex_candles.uses_gecko_candles(connector): - candles = await _fetch_dex_candles( - connector, pool_address, trading_pair, interval, start_time, end_time - ) - _candle_cache_put(cache_key, candles, now) - return candles - - client = await cm.get_client(name) - result = None - try: - # Prefer historical candles with time range when start_time is given - if start_time is not None: - st = int(start_time) - et = int(end_time) if end_time else int(time.time()) - logger.info( - "Fetching historical candles: connector=%s pair=%s interval=%s start=%s end=%s", - connector, - trading_pair, - interval, - st, - et, - ) - result = await client.market_data.get_historical_candles( - connector, - trading_pair, - interval, - start_time=st, - end_time=et, - ) - logger.info( - "Historical candles result: type=%s len=%s", - type(result).__name__, - len(result) if isinstance(result, (list, dict)) else "?", - ) - except Exception as e: - logger.warning( - "get_historical_candles failed: %s — falling back to get_candles", e - ) - result = None - - # Fallback: if historical returned nothing usable, use regular candles - candles_raw = ( - result - if isinstance(result, list) - else result.get("data", []) if isinstance(result, dict) else [] + # Cache miss: share one upstream fetch with every concurrent request on this + # key. Reuse an in-flight task only from the loop that created it — a task is + # bound to its loop and awaiting it from another one raises. + loop = asyncio.get_running_loop() + inflight = _candle_inflight.get(cache_key) + task = ( + inflight[1] + if inflight is not None and inflight[0] is loop and not inflight[1].done() + # A finished task lingers until its done-callback runs; joining it would + # hand a fresh request the previous fetch's outcome (a failure included). + else None ) - if not candles_raw: - try: - logger.info( - "Falling back to get_candles: connector=%s pair=%s interval=%s limit=%s", + if task is None: + task = asyncio.ensure_future( + _fetch_candles_upstream( + cm, + name, connector, trading_pair, interval, limit, + start_time, + end_time, + pool_address, ) - result = await client.market_data.get_candles( - connector, trading_pair, interval, limit - ) - except Exception as e: - raise HTTPException(status_code=502, detail=str(e)) - - candles_raw = ( - result - if isinstance(result, list) - else result.get("data", []) if isinstance(result, dict) else [] - ) + ) + _candle_inflight[cache_key] = (loop, task) + task.add_done_callback(lambda _t, k=cache_key: _candle_inflight.pop(k, None)) - candles = [] - for c in candles_raw: - if isinstance(c, dict): - candles.append( - CandleData( - timestamp=float(c.get("timestamp", 0)), - open=float(c.get("open", 0)), - high=float(c.get("high", 0)), - low=float(c.get("low", 0)), - close=float(c.get("close", 0)), - volume=float(c.get("volume", 0)), - ) - ) - elif isinstance(c, (list, tuple)) and len(c) >= 6: - candles.append( - CandleData( - timestamp=float(c[0]), - open=float(c[1]), - high=float(c[2]), - low=float(c[3]), - close=float(c[4]), - volume=float(c[5]), - ) - ) + # Shielded: a browser that disconnects mid-request cancels this handler, and + # an unshielded `await task` would cancel the shared fetch out from under + # every other waiter on the same key. + candles = await asyncio.shield(task) _candle_cache_put(cache_key, candles, now) return candles @@ -462,7 +559,7 @@ async def get_token_symbol( executor tables can resolve a symbol without threading a server name through every row. Auth is still required. """ - from handlers.dex.pool_data import fetch_token_symbol + from condor.pool_data import fetch_token_symbol if not _ADDRESS_RE.match(mint): raise HTTPException(status_code=400, detail="Invalid token address") diff --git a/condor/web/routes/meta.py b/condor/web/routes/meta.py new file mode 100644 index 000000000..bf5003cd5 --- /dev/null +++ b/condor/web/routes/meta.py @@ -0,0 +1,37 @@ +"""Build identity for the dashboard. + +One endpoint, and it exists for a single reason: a bug report that does not say +which commit it came from costs a round trip to find out. The fields are the +same ones the telemetry context already computes for its own envelope — a short +commit, a branch, a runtime — so nothing new is collected here, it is only shown +to the logged-in user who is about to paste it into an issue. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends + +from condor.telemetry import context +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +router = APIRouter(prefix="/meta", tags=["meta"]) + + +@router.get("/env") +async def get_env(user: WebUser = Depends(get_current_user)) -> dict: + """Version and platform of this install. + + Authenticated: the commit an install runs is not a secret, but it is a + detail about someone's deployment and there is no reason to hand it to an + anonymous caller. + """ + app = context.app() + return { + "version": app["version"], + "branch": app["branch"], + "python": app["python"], + "os": app["os"], + "arch": app["arch"], + "in_docker": app["in_docker"], + } diff --git a/condor/web/routes/portfolio.py b/condor/web/routes/portfolio.py index cf7291315..19b5829e6 100644 --- a/condor/web/routes/portfolio.py +++ b/condor/web/routes/portfolio.py @@ -1,15 +1,13 @@ from __future__ import annotations -import asyncio import logging -import time -from dataclasses import dataclass from datetime import datetime, timezone from typing import Any from fastapi import APIRouter, Depends, HTTPException, Query -from condor.web.auth import get_current_user +from condor.fetchers.portfolio import PORTFOLIO_HISTORY_RANGES +from condor.web.auth import require_server_access from condor.web.models import ( BalanceItem, ConnectorBalance, @@ -24,43 +22,14 @@ router = APIRouter(tags=["portfolio"]) -# --------------------------------------------------------------------------- -# Portfolio history cache -# --------------------------------------------------------------------------- - -_RANGE_TTLS: dict[str, int] = { - "1D": 120, - "1W": 600, - "1M": 3600, - "3M": 7200, -} - -_WARM_INTERVAL = 120 # Re-warm every 2 min (matches shortest TTL) -_IDLE_TIMEOUT = 600 # Stop refresh loop after 10 min with no requests - - -@dataclass -class _CacheEntry: - data: Any = None - fetched_at: float = 0.0 - ttl: int = 120 - - -# (server, range, False) -> CacheEntry (breakdown doesn't affect fetched data) -_history_cache: dict[tuple[str, str, bool], _CacheEntry] = {} -_refresh_tasks: dict[str, asyncio.Task] = {} -_last_request_time: dict[str, float] = {} # server -> last request ts - @router.get("/servers/{name}/portfolio", response_model=PortfolioResponse) async def get_portfolio( name: str, refresh: bool = Query(False), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access to this server") from condor.server_data_service import ServerDataType, get_server_data_service @@ -196,106 +165,6 @@ def _dedupe_hyperliquid_unified(connectors: list[ConnectorBalance]) -> None: ) -RANGE_CONFIG = { - "1D": (86400, "5m"), - "1W": (604800, "1h"), - "1M": (2592000, "4h"), - "3M": (7776000, "1d"), -} - - -async def _fetch_history(server: str, range_key: str) -> Any: - """Fetch portfolio history from the Hummingbot API (no caching).""" - cm = get_config_manager() - client = await cm.get_client(server) - range_seconds, interval = RANGE_CONFIG[range_key] - start_time = int(time.time()) - range_seconds - return await client.portfolio.get_history( - start_time=start_time, interval=interval, limit=500 - ) - - -async def _get_cached_history( - server: str, range_key: str, breakdown: bool = False -) -> Any: - """Return cached history if fresh, otherwise fetch, cache, and return. - - The raw data is the same regardless of *breakdown* (parsing happens in the - endpoint), so we cache without the breakdown flag to avoid duplicate fetches. - """ - key = (server, range_key, False) - entry = _history_cache.get(key) - now = time.time() - - if entry and entry.data is not None and (now - entry.fetched_at) < entry.ttl: - return entry.data - - # Fetch fresh data - data = await _fetch_history(server, range_key) - _history_cache[key] = _CacheEntry( - data=data, fetched_at=time.time(), ttl=_RANGE_TTLS[range_key] - ) - return data - - -async def warm_portfolio_history(server: str) -> None: - """Pre-fetch all 4 history ranges into cache, then start a refresh loop.""" - logger.info("Warming portfolio history cache for %s", server) - _last_request_time[server] = time.time() - - # Initial warm: fetch all ranges concurrently (no breakdown) - tasks = [] - for range_key in RANGE_CONFIG: - tasks.append(_get_cached_history(server, range_key, False)) - results = await asyncio.gather(*tasks, return_exceptions=True) - for range_key, result in zip(RANGE_CONFIG, results): - if isinstance(result, Exception): - logger.warning("Warm cache failed for %s/%s: %s", server, range_key, result) - - # Start background refresh loop if not already running - if server not in _refresh_tasks or _refresh_tasks[server].done(): - _refresh_tasks[server] = asyncio.create_task(_refresh_loop(server)) - - -async def _refresh_loop(server: str) -> None: - """Periodically re-warm cache; exits after idle timeout.""" - logger.info("Portfolio history refresh loop started for %s", server) - try: - while True: - await asyncio.sleep(_WARM_INTERVAL) - last = _last_request_time.get(server, 0) - if time.time() - last > _IDLE_TIMEOUT: - logger.info( - "Portfolio history refresh loop idle for %s, stopping", server - ) - return - - tasks = [] - for range_key in RANGE_CONFIG: - tasks.append(_fetch_history(server, range_key)) - results = await asyncio.gather(*tasks, return_exceptions=True) - for range_key, result in zip(RANGE_CONFIG, results): - if isinstance(result, Exception): - logger.debug( - "Refresh failed for %s/%s: %s", server, range_key, result - ) - continue - key = (server, range_key, False) - _history_cache[key] = _CacheEntry( - data=result, fetched_at=time.time(), ttl=_RANGE_TTLS[range_key] - ) - except asyncio.CancelledError: - logger.info("Portfolio history refresh loop cancelled for %s", server) - - -def stop_history_refresh(server: str) -> None: - """Cancel the background refresh loop for a server.""" - task = _refresh_tasks.pop(server, None) - if task and not task.done(): - task.cancel() - logger.info("Stopped portfolio history refresh for %s", server) - - @router.get( "/servers/{name}/portfolio/history", response_model=PortfolioHistoryResponse ) @@ -303,22 +172,23 @@ async def get_portfolio_history( name: str, range: str = Query("1D", pattern="^(1D|1W|1M|3M)$"), breakdown: bool = Query(False), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): - cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access to this server") - _last_request_time[name] = time.time() + from condor.server_data_service import ServerDataType, get_server_data_service + # The raw snapshots are the same regardless of *breakdown* (parsing happens + # below), so the cache key is the range alone. SDS coalesces concurrent + # reads of the same expired key into one backend fetch. try: - history = await _get_cached_history(name, range, breakdown) + history = await get_server_data_service().get_or_fetch( + name, ServerDataType.PORTFOLIO_HISTORY, range_key=range + ) except Exception as e: logger.warning("Failed to get portfolio history: %s", e) - _, interval = RANGE_CONFIG[range] - return PortfolioHistoryResponse(server=name, points=[], interval=interval) + history = None - _, interval = RANGE_CONFIG[range] + _, interval = PORTFOLIO_HISTORY_RANGES[range] logger.debug("Portfolio history response shape: %s", type(history)) if isinstance(history, dict): diff --git a/condor/web/routes/positions.py b/condor/web/routes/positions.py index fa4abb96f..d7baee46a 100644 --- a/condor/web/routes/positions.py +++ b/condor/web/routes/positions.py @@ -3,11 +3,11 @@ import asyncio import logging -from fastapi import APIRouter, Depends, HTTPException +from fastapi import APIRouter, Depends -from config_manager import get_config_manager -from condor.web.auth import get_current_user +from condor.web.auth import require_server_access from condor.web.models import WebUser +from config_manager import get_config_manager logger = logging.getLogger(__name__) @@ -30,7 +30,9 @@ def _normalize_position(pos: dict, source: str, source_name: str) -> dict: "entry_price": entry_price, "notional_value": notional_value, "current_price": pos.get("current_price") or 0, - "unrealized_pnl": pos.get("unrealized_pnl_quote") or pos.get("unrealized_pnl") or 0, + "unrealized_pnl": pos.get("unrealized_pnl_quote") + or pos.get("unrealized_pnl") + or 0, "leverage": pos.get("leverage") or 1, "controller_id": pos.get("controller_id") or "", "realized_pnl": pos.get("realized_pnl_quote") or 0, @@ -44,11 +46,9 @@ def _normalize_position(pos: dict, source: str, source_name: str) -> dict: @router.get("/servers/{name}/positions") async def get_consolidated_positions( name: str, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access), ): cm = get_config_manager() - if not cm.has_server_access(user.id, name): - raise HTTPException(status_code=403, detail="No access") from condor.server_data_service import ServerDataType, get_server_data_service @@ -72,7 +72,9 @@ async def fetch_executor_positions(): async def fetch_bot_positions(): try: - result = await get_server_data_service().get_or_fetch(name, ServerDataType.BOTS_STATUS) + result = await get_server_data_service().get_or_fetch( + name, ServerDataType.BOTS_STATUS + ) if result is None: return [] @@ -81,7 +83,11 @@ async def fetch_bot_positions(): if isinstance(result, dict): data = result.get("data", {}) if isinstance(data, dict): - bots_list = [{"bot_name": k, **v} for k, v in data.items() if isinstance(v, dict)] + bots_list = [ + {"bot_name": k, **v} + for k, v in data.items() + if isinstance(v, dict) + ] elif isinstance(data, list): bots_list = [b for b in data if isinstance(b, dict)] elif isinstance(result, list): @@ -121,8 +127,7 @@ async def fetch_bot_positions(): ] bot_positions = [ - _normalize_position(pos, "bot", source_name) - for pos, source_name in bot_raw + _normalize_position(pos, "bot", source_name) for pos, source_name in bot_raw ] # Enrich positions missing current_price (the positions_summary endpoint doesn't provide it) diff --git a/condor/web/routes/reports.py b/condor/web/routes/reports.py index ae62b2ab7..af3ed597b 100644 --- a/condor/web/routes/reports.py +++ b/condor/web/routes/reports.py @@ -13,7 +13,7 @@ list_reports_grouped, ) from condor.web.auth import get_current_user -from condor.web.models import ReportSummary, ReportsListResponse, WebUser +from condor.web.models import ReportsListResponse, ReportSummary, WebUser router = APIRouter(prefix="/reports", tags=["reports"]) diff --git a/condor/web/routes/routines.py b/condor/web/routes/routines.py index f406d4010..5af572a89 100644 --- a/condor/web/routes/routines.py +++ b/condor/web/routes/routines.py @@ -14,7 +14,12 @@ from condor.reports import list_reports from condor.routine_store import get_routine_store from condor.runtime import client -from condor.web.auth import get_current_user +from condor.web.auth import ( + check_server_access, + get_current_user, + require_server_access_by_server_name, + require_server_access_query, +) from condor.web.models import WebUser from config_manager import get_config_manager @@ -65,6 +70,40 @@ class HooksRequest(BaseModel): trigger: str = "success" +# ── Helpers ── + + +def _owns(inst: dict, user: WebUser) -> bool: + """Whether ``user`` started this instance. + + The owner is the ``user_id`` RoutineStore records in the instance meta. + Telegram-started instances now carry it too (CORR-165: the starting user, + not the chat). An instance without one is nobody's — MCP runs that carried + no user, or meta written by an older build — and treating them as unowned + keeps them out of every non-admin's reach instead of handing them to + whoever asks first. + """ + owner = inst.get("user_id") + return bool(owner) and owner == user.id + + +def _require_instance_access(inst: dict, user: WebUser) -> None: + """Admins reach every instance; everyone else only their own (SEC-150).""" + if get_config_manager().is_admin(user.id): + return + if not _owns(inst, user): + raise HTTPException(status_code=403, detail="Not your instance") + + +def _authorized_instance(instance_id: str, user: WebUser) -> dict: + """Fetch an instance, 404 if absent and 403 if it belongs to someone else.""" + inst = get_routine_store().get_instance(instance_id) + if not inst: + raise HTTPException(404, "Instance not found") + _require_instance_access(inst, user) + return inst + + # ── Routes ── @@ -77,19 +116,18 @@ async def list_routines(user: WebUser = Depends(get_current_user)): @router.get("/instances") async def list_instances(user: WebUser = Depends(get_current_user)): - """List all active routine instances.""" + """List the caller's active routine instances (admins see them all).""" store = get_routine_store() - return store.list_instances() + instances = store.list_instances() + if get_config_manager().is_admin(user.id): + return instances + return [inst for inst in instances if _owns(inst, user)] @router.get("/instances/{instance_id}") async def get_instance(instance_id: str, user: WebUser = Depends(get_current_user)): """Get instance detail including last result.""" - store = get_routine_store() - inst = store.get_instance(instance_id) - if not inst: - raise HTTPException(404, "Instance not found") - return inst + return _authorized_instance(instance_id, user) @router.get("/instances/{instance_id}/image") @@ -97,8 +135,8 @@ async def get_instance_image( instance_id: str, user: WebUser = Depends(get_current_user) ): """Serve the chart PNG for an instance result.""" - store = get_routine_store() - result = store.get_result(instance_id) + _authorized_instance(instance_id, user) + result = get_routine_store().get_result(instance_id) if not result or not result.chart_image: raise HTTPException(404, "No chart image available") return Response(content=result.chart_image, media_type="image/png") @@ -109,12 +147,9 @@ async def run_routine( server_name: str, routine_name: str, body: RunRequest, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_by_server_name), ): """Execute a one-shot routine. Returns instance_id for polling.""" - cm = get_config_manager() - if not cm.has_server_access(user.id, server_name): - raise HTTPException(status_code=403, detail="No access") store = get_routine_store() try: instance_id = await store.execute( @@ -133,12 +168,9 @@ async def schedule_routine( server_name: str, routine_name: str, body: ScheduleRequest, - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_by_server_name), ): """Schedule a routine at an interval. Returns instance_id.""" - cm = get_config_manager() - if not cm.has_server_access(user.id, server_name): - raise HTTPException(status_code=403, detail="No access") store = get_routine_store() try: instance_id = await store.schedule( @@ -159,9 +191,7 @@ async def run_routine_v2( user: WebUser = Depends(get_current_user), ): """Execute a routine (supports names with slashes like agent/routine).""" - cm = get_config_manager() - if not cm.has_server_access(user.id, body.server_name): - raise HTTPException(status_code=403, detail="No access") + check_server_access(user.id, body.server_name) store = get_routine_store() try: instance_id = await store.execute( @@ -189,9 +219,7 @@ async def start_continuous_v2( an agent starts from chat lives in the main process's store — visible in the dashboard and stoppable from it — instead of dying with the MCP subprocess. """ - cm = get_config_manager() - if not cm.has_server_access(user.id, body.server_name): - raise HTTPException(status_code=403, detail="No access") + check_server_access(user.id, body.server_name) store = get_routine_store() try: instance_id = await store.start_continuous( @@ -214,9 +242,7 @@ async def schedule_routine_v2( user: WebUser = Depends(get_current_user), ): """Schedule a routine (supports names with slashes like agent/routine).""" - cm = get_config_manager() - if not cm.has_server_access(user.id, body.server_name): - raise HTTPException(status_code=403, detail="No access") + check_server_access(user.id, body.server_name) store = get_routine_store() try: instance_id = await store.schedule( @@ -234,16 +260,20 @@ async def schedule_routine_v2( @router.post("/instances/{instance_id}/stop") async def stop_instance(instance_id: str, user: WebUser = Depends(get_current_user)): """Stop a running or scheduled instance.""" - store = get_routine_store() - if not store.stop(instance_id): + _authorized_instance(instance_id, user) + if not get_routine_store().stop(instance_id): raise HTTPException(404, "Instance not found") return {"stopped": True} @router.get("/{routine_name:path}/hooks") async def get_hooks(routine_name: str, user: WebUser = Depends(get_current_user)): - """Get the post-execution hook config for a routine.""" - cfg = routine_hooks.load_hooks(routine_name) + """Get the caller's post-execution hook config for a routine. + + Hooks are per-owner (SEC-152), so this never exposes another user's + delivery destinations: a caller who configured none gets the empty default. + """ + cfg = routine_hooks.load_hooks(routine_name, user.id) return cfg if cfg is not None else routine_hooks._default_config() @@ -253,17 +283,26 @@ async def put_hooks( body: HooksRequest, user: WebUser = Depends(get_current_user), ): - """Save the post-execution hook config for a routine.""" - return routine_hooks.save_hooks(routine_name, body.model_dump()) + """Save the caller's post-execution hook config for a routine.""" + try: + return routine_hooks.save_hooks(routine_name, body.model_dump(), user.id) + except routine_hooks.ForbiddenRecipient as e: + raise HTTPException(status_code=403, detail=str(e)) @router.get("/options/{source}") async def get_field_options( source: str, - server: str = Query("local", alias="server"), - user: WebUser = Depends(get_current_user), + server: str = Query(...), + user: WebUser = Depends(require_server_access_query), ): - """Return dynamic options for routine config fields (e.g. controller_configs).""" + """Return dynamic options for routine config fields (e.g. controller_configs). + + Server-scoped: ``controller_configs`` reads the named server's controller + configs, so the caller must have access to it (SEC-159). ``server`` used to + default to ``"local"``; the guard makes it required, which is what the + dashboard has always sent anyway. + """ if source == "controller_configs": try: cm = get_config_manager() diff --git a/condor/web/routes/servers.py b/condor/web/routes/servers.py index 03d87b42a..9d94afd91 100644 --- a/condor/web/routes/servers.py +++ b/condor/web/routes/servers.py @@ -1,29 +1,56 @@ from __future__ import annotations +import asyncio +import logging + from fastapi import APIRouter, Depends, HTTPException -from condor.server_data_service import ServerDataType, get_server_data_service +from condor.server_data_service import ( + ServerDataService, + ServerDataType, + get_server_data_service, +) from condor.web.auth import get_current_user from condor.web.models import ServerInfo, WebUser from config_manager import get_config_manager +logger = logging.getLogger(__name__) + router = APIRouter(tags=["servers"]) +async def _resolve_online(sds: ServerDataService, name: str) -> bool: + """Resolve one server's online flag, never raising. + + An unreachable server caches an error-only entry (value=None), so ``get()`` + misses on every request and the fetch pays a full connect timeout. Callers + run these concurrently, so a single failure must not sink the whole batch. + """ + try: + status = sds.get(name, ServerDataType.SERVER_STATUS) + if status is None: + status = await sds.get_or_fetch(name, ServerDataType.SERVER_STATUS) + except Exception as e: # noqa: BLE001 - a broken server is offline, not a 500 + logger.debug("servers: status fetch failed for %s: %s", name, e) + return False + return status.get("status") == "online" if isinstance(status, dict) else False + + @router.get("/servers", response_model=list[ServerInfo]) async def list_servers(user: WebUser = Depends(get_current_user)): cm = get_config_manager() accessible = cm.list_accessible_servers(user.id) sds = get_server_data_service() + # Per-server status resolution is independent and each miss can cost a full + # connect timeout, so fan it out: the endpoint waits once, not N times. + names = list(accessible) + flags = await asyncio.gather(*(_resolve_online(sds, name) for name in names)) + results = [] - for name, cfg in accessible.items(): + for name, online in zip(names, flags): + cfg = accessible[name] perm = cm.get_server_permission(user.id, name) - # Try cache first, fetch if missing/expired - status = sds.get(name, ServerDataType.SERVER_STATUS) - if status is None: - status = await sds.get_or_fetch(name, ServerDataType.SERVER_STATUS) - online = status.get("status") == "online" if isinstance(status, dict) else False results.append( ServerInfo( name=name, diff --git a/condor/web/routes/sessions.py b/condor/web/routes/sessions.py index 102dffb93..d2a484605 100644 --- a/condor/web/routes/sessions.py +++ b/condor/web/routes/sessions.py @@ -22,7 +22,7 @@ from condor.runtime import conversations from condor.runtime.binding import UnknownAgent, remember_model_choice from condor.runtime.sse import SSE_HEADERS, event_stream -from condor.web.auth import get_current_user +from condor.web.auth import check_server_access, get_current_user from condor.web.models import WebUser from config_manager import get_config_manager @@ -146,6 +146,15 @@ async def create_session( elif spec.user_id != user.id and not cm.is_admin(user.id): raise HTTPException(status_code=403, detail="Cannot create for another user") + # The session's toolset is built with this server's API credentials, so gate + # the pinned name exactly like ``_respawn`` does. The subject is the + # *caller*, never ``spec.user_id`` (SEC-167): an admin creating a session + # for someone else is held to their own reach, not licensed by the owner's. + # The rest of the spec is not a licence either — ``spec.chat_id`` reaches + # server resolution too, and is checked where it lands (SEC-178). + if spec.server_name: + check_server_access(user.id, spec.server_name) + from condor.preferences import load_user_data_for try: @@ -227,12 +236,14 @@ async def session_action( return {"ok": await runtime.abort(parsed)} if body.action in ("new", "switch"): - return await _respawn(parsed, info, body) + return await _respawn(parsed, info, body, user) raise HTTPException(status_code=400, detail=f"Unknown action: {body.action}") -async def _respawn(key: SessionKey, info: SessionInfo, body: SessionAction) -> dict: +async def _respawn( + key: SessionKey, info: SessionInfo, body: SessionAction, user: WebUser +) -> dict: """Tear the session down and bring it back under the same key. ACP has no identity hot-swap, so switching who is answering means reaping @@ -249,14 +260,25 @@ async def _respawn(key: SessionKey, info: SessionInfo, body: SessionAction) -> d switching = body.action == "switch" + # A respawn *creates* a session, so it needs a real owner to create it for: + # the new spec's ``user_id``, the user_data loaded for it, the remembered + # model choice and the conversation it writes into are all keyed by that id. + # A record carrying none has no such subject, so refuse rather than stand a + # placeholder id in for it — the same fail-closed treatment SEC-150 gave + # ownerless routine instances. Only an admin can get here with one anyway: + # ``_require_ownership`` already refuses everyone else. + if info.user_id is None: + raise HTTPException(status_code=403, detail="Session has no owner") + # The new session's tools are built against this server's credentials, so # gate it exactly like consult/delegate do — otherwise a switch would be a - # way to reach a server the caller was never granted (IDOR). Checked before - # the teardown, so a refused switch leaves the session running as it was. - if body.server_name and not get_config_manager().has_server_access( - info.user_id or 0, body.server_name - ): - raise HTTPException(status_code=403, detail="No access") + # way to reach a server the caller was never granted (IDOR). The subject is + # the *caller*, never the session's owner (SEC-167): an admin acting on + # someone else's session must be held to their own reach, not licensed by + # the owner's. Checked before the teardown, so a refused switch leaves the + # session running as it was. + if body.server_name: + check_server_access(user.id, body.server_name) # Unspecified fields keep their current value, so "new" is just a switch # to the same identity. diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index 63b0cdf2b..d996f5610 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -5,18 +5,22 @@ from fastapi import APIRouter, Depends, HTTPException, Query -from config_manager import ServerPermission, get_config_manager -from condor.web.auth import get_current_user +from condor.web.auth import get_current_user, require_server_access_query from condor.web.models import ( AddCredentialRequest, AddServerRequest, CredentialInfo, + GatewayNetworkUpdateRequest, GatewayPullRequest, GatewayStartRequest, + GatewayWalletAddRequest, + GatewayWalletDefaultRequest, ServerInfo, UpdateServerRequest, WebUser, ) +from condor.web.routes._errors import upstream_error +from config_manager import ServerPermission, get_config_manager logger = logging.getLogger(__name__) @@ -27,6 +31,22 @@ def _require_owner(cm, user_id: int, server_name: str): + """Enforce the OWNER line on top of the TRADER floor every route already has. + + The rule this file draws (SEC-153, extended to the gateway by SEC-166): + + * **Reading** a server's state — status, logs, wallet and network listings, + configured connectors — needs TRADER. A shared trader has to be able to + see what they are trading against, and to tell the owner when it is down. + * **Trading** on a server needs TRADER. That is what the share is for. + * **Mutating a server's configuration or its infrastructure** needs OWNER. + Exchange credentials, the gateway container lifecycle, the private keys + in its keystore and the RPC endpoints it dials are all the owner's + machine, not a trading action — and each of them can break the owner's + running bots for everyone else on the server. + + Admins keep the bypass they hold everywhere else in this module. + """ perm = cm.get_server_permission(user_id, server_name) if perm != ServerPermission.OWNER and not cm.is_admin(user_id): raise HTTPException(status_code=403, detail="Owner access required") @@ -35,8 +55,15 @@ def _require_owner(cm, user_id: int, server_name: str): async def _get_client(cm, server_name: str): try: return await cm.get_client(server_name) - except Exception as e: + except ValueError as e: + # The config manager's own rejection ("no such server"). Its text names + # nothing the caller did not already type, so it can be shown as-is. raise HTTPException(status_code=502, detail=f"Cannot connect to server: {e}") + except Exception as e: + # Anything else came off the wire, and its string carries the backend + # address — log it, show the caller only the safe description. + logger.exception("Cannot connect to server '%s'", server_name) + raise upstream_error("Cannot connect to server", e) # ── Servers ── @@ -48,6 +75,7 @@ async def list_settings_servers(user: WebUser = Depends(get_current_user)): accessible = cm.list_accessible_servers(user.id) from condor.server_data_service import ServerDataType, get_server_data_service + sds = get_server_data_service() # Fetch status for all servers concurrently (uses SDS cache, instant if warm) @@ -61,13 +89,15 @@ async def _get_status(name: str) -> dict: for (name, cfg), status in zip(accessible.items(), statuses): perm = cm.get_server_permission(user.id, name) online = status.get("status") == "online" - results.append(ServerInfo( - name=name, - host=cfg.get("host", ""), - port=cfg.get("port", 0), - online=online, - permission=perm.value if perm else "trader", - )) + results.append( + ServerInfo( + name=name, + host=cfg.get("host", ""), + port=cfg.get("port", 0), + online=online, + permission=perm.value if perm else "trader", + ) + ) return sorted(results, key=lambda s: (not s.online, s.name)) @@ -137,11 +167,9 @@ async def set_default_server(name: str, user: WebUser = Depends(get_current_user @router.get("/gateway/status") async def gateway_status( server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") client = await _get_client(cm, server) try: info = await client.gateway.get_status() @@ -162,11 +190,12 @@ async def gateway_status( async def gateway_pull( req: GatewayPullRequest, server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") + # Pulling an image rewrites what code will run on the owner's host next + # restart — infrastructure, not trading. + _require_owner(cm, user.id, server) client = await _get_client(cm, server) try: # Parse "image:tag" format (e.g. "hummingbot/gateway:latest") @@ -176,91 +205,297 @@ async def gateway_pull( result = await client.docker.pull_image(image_name, tag) return {"pulled": True, "image": req.image, "result": result} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logger.exception("Failed to pull gateway image '%s' on '%s'", req.image, server) + raise upstream_error("Failed to pull gateway image", e) @router.get("/gateway/pull-status") async def gateway_pull_status( server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") client = await _get_client(cm, server) try: result = await client.docker.get_pull_status() return result except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logger.exception("Failed to fetch gateway pull status from '%s'", server) + raise upstream_error("Failed to fetch pull status", e) @router.post("/gateway/start") async def gateway_start( req: GatewayStartRequest, server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") + # The caller picks the image and port of a container started on the owner's + # host. That is the owner's decision, not a shared trader's. + _require_owner(cm, user.id, server) client = await _get_client(cm, server) try: - result = await client.gateway.start({ - "image": req.image, - "port": req.port, - }) + result = await client.gateway.start( + { + "image": req.image, + "port": req.port, + } + ) return {"started": True, "result": result} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logger.exception("Failed to start gateway on '%s'", server) + raise upstream_error("Failed to start gateway", e) @router.post("/gateway/stop") async def gateway_stop( server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") + # Stopping the gateway cuts DEX connectivity for every bot on the server, + # including the owner's. Same line delete_credential draws. + _require_owner(cm, user.id, server) client = await _get_client(cm, server) try: result = await client.gateway.stop() return {"stopped": True, "result": result} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logger.exception("Failed to stop gateway on '%s'", server) + raise upstream_error("Failed to stop gateway", e) @router.post("/gateway/restart") async def gateway_restart( server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") + # A restart drops in-flight DEX orders for everyone on the server. + _require_owner(cm, user.id, server) client = await _get_client(cm, server) try: result = await client.gateway.restart() return {"restarted": True, "result": result} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logger.exception("Failed to restart gateway on '%s'", server) + raise upstream_error("Failed to restart gateway", e) @router.get("/gateway/logs") async def gateway_logs( server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") client = await _get_client(cm, server) try: logs = await client.gateway.get_logs() return {"logs": logs} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logger.exception("Failed to fetch gateway logs from '%s'", server) + raise upstream_error("Failed to fetch gateway logs", e) + + +# ── Gateway Networks (RPC) ── + + +@router.get("/gateway/networks") +async def gateway_networks( + server: str = Query(...), + user: WebUser = Depends(require_server_access_query), +): + cm = get_config_manager() + client = await _get_client(cm, server) + try: + result = await client.gateway.list_networks() + networks = ( + result.get("networks", result) if isinstance(result, dict) else result + ) + return {"networks": networks} + except Exception as e: + logger.exception("Failed to list gateway networks from '%s'", server) + raise upstream_error("Failed to list gateway networks", e) + + +@router.get("/gateway/networks/{network_id}") +async def gateway_network_config( + network_id: str, + server: str = Query(...), + user: WebUser = Depends(require_server_access_query), +): + cm = get_config_manager() + client = await _get_client(cm, server) + try: + config = await client.gateway.get_network_config(network_id) + return {"network_id": network_id, "config": config} + except Exception as e: + logger.exception( + "Failed to fetch gateway network config '%s' from '%s'", network_id, server + ) + raise upstream_error("Failed to fetch network config", e) + + +@router.post("/gateway/networks/{network_id}") +async def gateway_network_update( + network_id: str, + req: GatewayNetworkUpdateRequest, + server: str = Query(...), + user: WebUser = Depends(require_server_access_query), +): + cm = get_config_manager() + # Network config carries the RPC endpoint every transaction from this server + # is signed and broadcast through — repointing it is a server-wide change. + _require_owner(cm, user.id, server) + client = await _get_client(cm, server) + try: + result = await client.gateway.update_network_config(network_id, req.config) + return {"updated": True, "network_id": network_id, "result": result} + except Exception as e: + logger.exception( + "Failed to update gateway network config '%s' on '%s'", network_id, server + ) + raise upstream_error("Failed to update network config", e) + + +# ── Gateway Wallets ── + + +@router.get("/gateway/wallets") +async def gateway_wallets( + server: str = Query(...), + user: WebUser = Depends(require_server_access_query), +): + cm = get_config_manager() + client = await _get_client(cm, server) + try: + wallets = await client.accounts.list_gateway_wallets() + return {"wallets": wallets} + except Exception as e: + logger.exception("Failed to list gateway wallets from '%s'", server) + raise upstream_error("Failed to list gateway wallets", e) + + +async def _gateway_default_address(client, chain: str) -> str | None: + """Current default wallet address for a chain, or None if there is none.""" + try: + groups = await client.accounts.list_gateway_wallets() + except Exception: + logger.warning( + "Could not read gateway wallets to preserve default for '%s'", chain + ) + return None + for group in groups or []: + if isinstance(group, dict) and group.get("chain") == chain: + return group.get("default_address") or None + return None + + +@router.post("/gateway/wallets") +async def gateway_wallet_add( + req: GatewayWalletAddRequest, + server: str = Query(...), + user: WebUser = Depends(require_server_access_query), +): + cm = get_config_manager() + # Importing a private key into the owner's gateway makes that key usable by + # everyone on the server. Whose keys live on the host is the owner's call. + _require_owner(cm, user.id, server) + client = await _get_client(cm, server) + # Gateway's add-wallet always promotes the new key to the chain default, so when the user + # opted out we capture the existing default first and restore it after the import. + previous_default = ( + None if req.set_default else await _gateway_default_address(client, req.chain) + ) + try: + result = await client.accounts.add_gateway_wallet( + chain=req.chain, private_key=req.private_key + ) + address = result.get("address") if isinstance(result, dict) else None + is_default = True + if req.set_default and address: + await client.accounts.set_default_gateway_wallet( + chain=req.chain, address=address + ) + elif previous_default and previous_default != address: + try: + await client.accounts.set_default_gateway_wallet( + chain=req.chain, address=previous_default + ) + is_default = False + except Exception: + logger.exception( + "Imported wallet on '%s' but failed to restore previous default on chain '%s'", + server, + req.chain, + ) + return { + "added": True, + "chain": req.chain, + "address": address, + "is_default": is_default, + } + except Exception as e: + # Never include the request payload here — it carries the private key. + logger.exception( + "Failed to add gateway wallet on chain '%s' via '%s'", req.chain, server + ) + raise upstream_error("Failed to add wallet", e) + + +@router.post("/gateway/wallets/default") +async def gateway_wallet_set_default( + req: GatewayWalletDefaultRequest, + server: str = Query(...), + user: WebUser = Depends(require_server_access_query), +): + cm = get_config_manager() + # The default wallet is the one every subsequent DEX trade signs with, so + # switching it silently redirects the owner's flow to another address. + _require_owner(cm, user.id, server) + client = await _get_client(cm, server) + try: + result = await client.accounts.set_default_gateway_wallet( + chain=req.chain, address=req.address + ) + return { + "default": True, + "chain": req.chain, + "address": req.address, + "result": result, + } + except Exception as e: + logger.exception( + "Failed to set default gateway wallet on '%s' via '%s'", req.chain, server + ) + raise upstream_error("Failed to set default wallet", e) + + +@router.delete("/gateway/wallets/{chain}/{address}") +async def gateway_wallet_remove( + chain: str, + address: str, + server: str = Query(...), + user: WebUser = Depends(require_server_access_query), +): + cm = get_config_manager() + # Removing a key strands any bot trading from that address — owner only. + _require_owner(cm, user.id, server) + client = await _get_client(cm, server) + try: + result = await client.accounts.remove_gateway_wallet( + chain=chain, address=address + ) + return {"removed": True, "chain": chain, "address": address, "result": result} + except Exception as e: + logger.exception( + "Failed to remove gateway wallet %s on '%s' via '%s'", + address, + chain, + server, + ) + raise upstream_error("Failed to remove wallet", e) # ── Voice Preferences ── @@ -271,12 +506,16 @@ async def get_voice_settings(user: WebUser = Depends(get_current_user)): """Get voice/transcription preferences for the current user.""" cm = get_config_manager() prefs = cm.get_user_preferences(user.id) - voice = prefs.get("voice", { - "whisper_model": "small", - "language": None, - "auto_send": True, - }) - from condor.preferences import WHISPER_MODELS, VOICE_LANGUAGES + voice = prefs.get( + "voice", + { + "whisper_model": "small", + "language": None, + "auto_send": True, + }, + ) + from condor.preferences import VOICE_LANGUAGES, WHISPER_MODELS + return { "voice": voice, "available_models": WHISPER_MODELS, @@ -292,11 +531,14 @@ async def update_voice_settings( """Update voice/transcription preferences.""" cm = get_config_manager() prefs = cm.get_user_preferences(user.id) - voice = prefs.get("voice", { - "whisper_model": "small", - "language": None, - "auto_send": True, - }) + voice = prefs.get( + "voice", + { + "whisper_model": "small", + "language": None, + "auto_send": True, + }, + ) allowed_keys = {"whisper_model", "language", "auto_send"} for key in allowed_keys: if key in body: @@ -304,6 +546,7 @@ async def update_voice_settings( # Validate whisper_model from condor.preferences import WHISPER_MODELS + if voice.get("whisper_model") not in WHISPER_MODELS: voice["whisper_model"] = "base" @@ -321,11 +564,9 @@ async def update_voice_settings( @router.get("/credentials") async def list_credentials( server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") client = await _get_client(cm, server) try: creds_raw = await client.accounts.list_account_credentials("master_account") @@ -336,40 +577,56 @@ async def list_credentials( if isinstance(item, str): credentials.append({"connector_name": item, "connector_type": ""}) elif isinstance(item, dict): - credentials.append({ - "connector_name": item.get("connector_name", item.get("name", "")), - "connector_type": item.get("connector_type", item.get("type", "")), - }) + credentials.append( + { + "connector_name": item.get( + "connector_name", item.get("name", "") + ), + "connector_type": item.get( + "connector_type", item.get("type", "") + ), + } + ) return {"credentials": credentials} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logger.exception("Failed to list credentials on '%s'", server) + raise upstream_error("Failed to list credentials", e) @router.get("/connectors") async def list_connectors( server: str = Query(...), type: str = Query(None), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): - cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") from condor.server_data_service import ServerDataType, get_server_data_service sds = get_server_data_service() raw = await sds.get_or_fetch(server, ServerDataType.ALL_CONNECTORS) if raw is None: - raise HTTPException(status_code=502, detail="Cannot fetch connectors from server") + raise HTTPException( + status_code=502, detail="Cannot fetch connectors from server" + ) # API returns plain strings — filter out testnet/gateway connectors - names = [c for c in raw if isinstance(c, str) and "testnet" not in c.lower() and "sandbox" not in c.lower() and "/" not in c] + names = [ + c + for c in raw + if isinstance(c, str) + and "testnet" not in c.lower() + and "sandbox" not in c.lower() + and "/" not in c + ] if type: if type.lower() == "perpetual": names = [c for c in names if "perpetual" in c.lower()] else: names = [c for c in names if "perpetual" not in c.lower()] - connectors = [{"name": c, "type": "perpetual" if "perpetual" in c.lower() else "spot"} for c in names] + connectors = [ + {"name": c, "type": "perpetual" if "perpetual" in c.lower() else "spot"} + for c in names + ] return {"connectors": connectors} @@ -377,28 +634,30 @@ async def list_connectors( async def connector_config_map( name: str, server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") client = await _get_client(cm, server) try: config_map = await client.connectors.get_config_map(name) return {"config_map": config_map} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logger.exception( + "Failed to fetch config map for connector '%s' on '%s'", name, server + ) + raise upstream_error("Failed to fetch connector config map", e) @router.post("/credentials") async def add_credential( req: AddCredentialRequest, server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") + # Credentials are server configuration, not trading: a shared trader must not + # overwrite the owner's exchange keys. Same line update_server/delete_server draw. + _require_owner(cm, user.id, server) client = await _get_client(cm, server) try: result = await client.accounts.add_credential( @@ -408,21 +667,25 @@ async def add_credential( ) # Invalidate configured connectors cache from condor.server_data_service import ServerDataType, get_server_data_service + get_server_data_service().invalidate(server, ServerDataType.CONNECTORS) return {"added": True, "result": result} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logger.exception( + "Failed to add credentials for '%s' on '%s'", req.connector_name, server + ) + raise upstream_error("Failed to add credentials", e) @router.delete("/credentials/{connector}") async def delete_credential( connector: str, server: str = Query(...), - user: WebUser = Depends(get_current_user), + user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() - if not cm.has_server_access(user.id, server): - raise HTTPException(status_code=403, detail="No access") + # Deleting a key kills the owner's running bots' connectivity — owner only. + _require_owner(cm, user.id, server) client = await _get_client(cm, server) try: result = await client.accounts.delete_credential( @@ -431,12 +694,16 @@ async def delete_credential( ) # Invalidate configured connectors + portfolio caches so the removed key disappears immediately from condor.server_data_service import ServerDataType, get_server_data_service + sds = get_server_data_service() sds.invalidate(server, ServerDataType.CONNECTORS) sds.invalidate(server, ServerDataType.PORTFOLIO) return {"deleted": True, "result": result} except Exception as e: - raise HTTPException(status_code=500, detail=str(e)) + logger.exception( + "Failed to delete credentials for '%s' on '%s'", connector, server + ) + raise upstream_error("Failed to delete credentials", e) # ── Custom OpenAI-compatible LLM endpoints ── @@ -550,3 +817,59 @@ async def delete_custom_provider( if not remove_custom_provider(load_user_data_for(user.id), name): raise HTTPException(status_code=404, detail=f"No saved endpoint '{name}'") return {"deleted": True, "name": name} + + +# ── Telemetry (FEAT-023) ── + + +@router.get("/telemetry") +async def get_telemetry_settings(user: WebUser = Depends(get_current_user)): + """What this install has agreed to, and whether it could send anything. + + ``endpoint_configured`` is the honest answer to "is this thing on": with no + ``CONDOR_TELEMETRY_URL`` set — the shipped state — nothing can be + transmitted no matter what the consent says, and events only accumulate in + a capped local file. + """ + from condor.telemetry import consent, emitter, outbox + + return { + "consent": consent.state(), + "level": consent.level(), + "env_overridden": consent.env_overridden(), + "endpoint_configured": bool(outbox.endpoint()), + "pending_events": emitter.buffered(), + "privacy_doc": "PRIVACY.md", + } + + +@router.put("/telemetry") +async def set_telemetry_settings( + level: str = Query(..., description="off | ping | usage"), + user: WebUser = Depends(get_current_user), +): + """Change the install's telemetry level. Admin only, and reversible. + + Setting ``off`` is a withdrawal, not a pause: the buffer and the outbox are + deleted, so nothing already recorded can be sent later. + """ + from condor.telemetry import consent + + cm = get_config_manager() + if not cm.is_admin(user.id): + raise HTTPException( + status_code=403, + detail="Telemetry is an install-wide setting; only the admin can change it", + ) + if level not in consent.LEVELS: + raise HTTPException( + status_code=400, detail=f"level must be one of {', '.join(consent.LEVELS)}" + ) + if consent.env_overridden(): + raise HTTPException( + status_code=409, + detail="CONDOR_TELEMETRY is set in the environment and overrides this setting", + ) + + applied = consent.set_level(level) + return {"level": applied, "consent": consent.state()} diff --git a/condor/web/routes/transcribe.py b/condor/web/routes/transcribe.py index 170898bea..fb534719f 100644 --- a/condor/web/routes/transcribe.py +++ b/condor/web/routes/transcribe.py @@ -2,7 +2,7 @@ import logging -from fastapi import APIRouter, Depends, Form, UploadFile, File, HTTPException +from fastapi import APIRouter, Depends, File, Form, HTTPException, UploadFile from condor.web.auth import get_current_user from condor.web.models import WebUser @@ -39,6 +39,7 @@ async def transcribe_audio( if effective_lang is None or effective_model is None: try: from config_manager import get_config_manager + cm = get_config_manager() voice_prefs = cm.get_user_preferences(user.id).get("voice", {}) if effective_lang is None: diff --git a/condor/web/ws_manager.py b/condor/web/ws_manager.py index ccfca1497..6122c5f37 100644 --- a/condor/web/ws_manager.py +++ b/condor/web/ws_manager.py @@ -5,13 +5,18 @@ import logging import math import time -from typing import Any, Awaitable, Callable, Optional +from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional from fastapi import WebSocket from condor import dex_candles +from condor.asyncutil import TaskSet +from condor.fetchers.market_data import fetch_historical_candles, normalize_candle from condor.web.auth import decode_jwt +if TYPE_CHECKING: + from condor.server_data_service import CacheKey + logger = logging.getLogger(__name__) # Mapping from WS channel prefix to ServerDataType @@ -22,13 +27,52 @@ "prices": "PRICES", } -# Reverse mapping for listener compatibility +# Reverse mapping, for naming the channel a cache write belongs to _SDT_TO_CHANNEL_PREFIX = { "PORTFOLIO": "portfolio", "BOTS_STATUS": "bots", "PRICES": "prices", } +# CacheKey params a data type's channel is scoped by, in channel-segment order. +# A data type absent from this table has a server-wide channel (`{prefix}:{server}`) +# and its key must carry no params at all — an account-scoped portfolio or any +# other parameterized variant is a *different* dataset and must not be broadcast +# to the server-wide channel. This is the single source of truth for both +# directions: channel -> key params when subscribing, key -> channel when +# broadcasting. +_CHANNEL_KEY_PARAMS: dict[str, tuple[str, ...]] = { + "PRICES": ("connector_name", "trading_pair"), +} + + +def channel_for_key(key: CacheKey) -> str | None: + """WS channel a written SDS ``CacheKey`` belongs to, or ``None`` if none does. + + ``None`` means "no channel carries this data" (a data type the dashboard does + not stream, e.g. portfolio history, or a parameterized variant of a + server-wide type) — never a silent drop of data a channel was waiting for: + a key that names a channel but is missing the params to address it is logged. + """ + prefix = _SDT_TO_CHANNEL_PREFIX.get(key.data_type.name) + if not prefix: + return None + + params = key.params_dict + param_names = _CHANNEL_KEY_PARAMS.get(key.data_type.name, ()) + if set(params) != set(param_names): + logger.debug( + "No %s channel for key with params %s (expected %s)", + prefix, + sorted(params), + list(param_names), + ) + return None + + segments = [params[name] for name in param_names] + return ":".join([prefix, key.server, *segments]) + + # Interval string -> seconds for buffer sizing _INTERVAL_SECONDS: dict[str, int] = { "1s": 1, @@ -68,6 +112,29 @@ def _coerce_duration(value: object) -> int | None: return duration if duration > 0 else None +def parse_candle_channel(channel: str) -> tuple[str, str, str, str, str | None] | None: + """Split a candle channel into its parts, or ``None`` if it is malformed. + + The channel is ``candles:{server}:{connector}:{pair}:{interval}[:{pool_address}]``. + The pool address is an **optional sixth segment**, set by the DEX page so a + chart is pinned to the exact pool the user opened instead of whichever pool + ``dex_candles._resolve_pool`` judges deepest for the pair. A five-part channel + (every CLOB chart) parses to ``pool_address=None`` and is byte-identical to + what it was before the segment existed. + + Every consumer must go through here: unpacking five names off ``split(":")`` + raises ``ValueError`` on a six-part channel, and each of the three call sites + fails silently in a different way (a stream that never backfills, a backfill + that never ticks). + """ + parts = channel.split(":") + if len(parts) < 5: + return None + server_name, connector, pair, interval = parts[1:5] + pool_address = parts[5] if len(parts) > 5 else None + return server_name, connector, pair, interval, pool_address or None + + class _CandleBuffer: """Per-channel candle buffer with dynamic sizing based on interval + duration.""" @@ -182,33 +249,12 @@ def __init__(self): # Strong refs to fire-and-forget one-shot tasks (backfill, warm cache) # so the GC can't cancel them mid-flight (the event loop only keeps a # weak reference). Entries auto-remove on completion. - self._oneshot_tasks: set[asyncio.Task] = set() + self._oneshot_tasks = TaskSet(logger, "One-shot task %s failed: %s") # Lazily-built registry of per-type stream lifecycles (see _stream_registry) self._stream_registry_cache: dict | None = None # -- Helpers -- - def _track_oneshot(self, task: asyncio.Task) -> None: - """Keep a strong reference to a fire-and-forget task until it finishes, - so the GC can't silently cancel it (the event loop only holds a weak - reference). The reference is dropped automatically on completion.""" - self._oneshot_tasks.add(task) - task.add_done_callback(self._on_oneshot_done) - - def _on_oneshot_done(self, task: asyncio.Task) -> None: - """Release the strong reference and surface failures on this module's - logger. Nobody awaits a one-shot task, so without this a crash would - only show up as asyncio's "Task exception was never retrieved" at GC - time, leaving a channel silently stalled.""" - self._oneshot_tasks.discard(task) - if task.cancelled(): - return - exc = task.exception() - if exc is not None: - logger.error( - "One-shot task %s failed: %s", task.get_name(), exc, exc_info=exc - ) - @staticmethod def _server_from_channel(channel: str) -> str | None: """Server name encoded as the second segment of a channel @@ -221,28 +267,7 @@ def _server_from_channel(channel: str) -> str | None: @staticmethod def _normalize_candle(c: Any) -> dict | None: """Normalize a candle from any format to a uniform dict with float values.""" - try: - if isinstance(c, dict): - return { - "timestamp": float(c.get("timestamp", 0)), - "open": float(c.get("open", 0)), - "high": float(c.get("high", 0)), - "low": float(c.get("low", 0)), - "close": float(c.get("close", 0)), - "volume": float(c.get("volume", 0)), - } - elif isinstance(c, (list, tuple)) and len(c) >= 6: - return { - "timestamp": float(c[0]), - "open": float(c[1]), - "high": float(c[2]), - "low": float(c[3]), - "close": float(c[4]), - "volume": float(c[5]), - } - except (TypeError, ValueError): - pass - return None + return normalize_candle(c) # -- Lifecycle -- @@ -286,12 +311,8 @@ def stop(self) -> None: self._last_candle_ws_update.clear() self._candle_first_msg_logged.clear() - # Cancel any still-pending one-shot tasks (snapshot: cancel() fires the - # done-callback that mutates the set). - for task in list(self._oneshot_tasks): - if not task.done(): - task.cancel() - self._oneshot_tasks.clear() + # Cancel any still-pending one-shot tasks. + self._oneshot_tasks.cancel_all() if self._cleanup_task and not self._cleanup_task.done(): self._cleanup_task.cancel() @@ -360,13 +381,12 @@ def _maybe_unsub_sds(self, channel: str) -> None: sds.unsubscribe(cache_key, "ws_manager") logger.debug("WS unsubscribed SDS for channel %s", channel) - # Stop portfolio history refresh when no subscribers remain + # The history ranges ride on this subscription — drop them too, or + # their polls would outlive the last dashboard client watching. if channel.startswith("portfolio:"): - from condor.web.routes.portfolio import stop_history_refresh - server_name = channel.split(":")[1] if ":" in channel else "" if server_name: - stop_history_refresh(server_name) + self._unsubscribe_portfolio_history(server_name) # -- Message handling -- @@ -456,7 +476,7 @@ async def _handle_candle_subscribe( old_max = buf.max_size buf.set_duration(dur) if buf.max_size > old_max and buf.needs_backfill: - self._track_oneshot( + self._oneshot_tasks.track( asyncio.create_task( self._backfill_candles(channel), name=f"backfill:{channel}", @@ -498,10 +518,10 @@ async def _handle_candle_duration_change( async def _backfill_candles(self, channel: str) -> None: """Fetch historical candles to fill the buffer gap.""" - parts = channel.split(":") - if len(parts) < 5: + parsed = parse_candle_channel(channel) + if parsed is None: return - _, server_name, connector, pair, interval = parts + server_name, connector, pair, interval, pool_address = parsed buf = self._candle_buffers.get(channel) if buf is None: return @@ -525,7 +545,7 @@ async def _backfill_candles(self, channel: str) -> None: if dex_candles.uses_gecko_candles(connector): candles = await dex_candles.fetch_dex_candles( - connector, None, pair, interval, start_time, end_time + connector, pool_address, pair, interval, start_time, end_time ) if candles: buf.upsert_many(candles) @@ -539,33 +559,15 @@ async def _backfill_candles(self, channel: str) -> None: return client = await cm.get_client(server_name) - result = await client.market_data.get_historical_candles( + candles = await fetch_historical_candles( + client, connector, pair, interval, start_time=start_time, end_time=end_time, + limit=min(buf.max_size, 5000), ) - - candles_raw = ( - result - if isinstance(result, list) - else result.get("data", []) if isinstance(result, dict) else [] - ) - # Fallback to regular candles if historical returned nothing - if not candles_raw: - result = await client.market_data.get_candles( - connector, pair, interval, min(buf.max_size, 5000) - ) - candles_raw = ( - result - if isinstance(result, list) - else result.get("data", []) if isinstance(result, dict) else [] - ) - - candles = [ - c for r in candles_raw if (c := self._normalize_candle(r)) is not None - ] if candles: buf.upsert_many(candles) logger.info( @@ -598,10 +600,12 @@ async def _subscribe_sds(self, channel: str) -> None: sds = get_server_data_service() data_type = ServerDataType[sdt_name] - # Build params for price channels - params = {} - if sdt_name == "PRICES" and len(parts) >= 4: - params = {"connector_name": parts[2], "trading_pair": parts[3]} + # Params the channel scopes its key by, read off the channel segments + # that follow the server (same table the broadcast direction uses). + param_names = _CHANNEL_KEY_PARAMS.get(sdt_name, ()) + if len(parts) < 2 + len(param_names): + return + params = dict(zip(param_names, parts[2:])) try: cache_key = await sds.subscribe( @@ -619,20 +623,81 @@ async def _subscribe_sds(self, channel: str) -> None: if result != prev: await self.broadcast(channel, result) - # Pre-warm portfolio history cache on subscription + # Portfolio history rides on the portfolio subscription: one SDS + # key per range, primed now and polled until the last client leaves. if prefix == "portfolio": - from condor.web.routes.portfolio import warm_portfolio_history - - self._track_oneshot( + self._oneshot_tasks.track( asyncio.create_task( - warm_portfolio_history(server_name), - name=f"warm_portfolio_history:{server_name}", + self._subscribe_portfolio_history(channel, server_name), + name=f"portfolio_history:{server_name}", ) ) except Exception as e: logger.debug("Failed to subscribe SDS for %s: %s", channel, e) - # -- SDS listener (legacy-compatible signature) -- + async def _subscribe_portfolio_history( + self, channel: str, server_name: str + ) -> None: + """Subscribe one SDS key per history range behind a portfolio channel. + + Runs off the subscribe path as a one-shot task: SDS primes each key with + a real backend fetch and the WS client must not wait for four of them. + Subscribing is idempotent per (key, subscriber), so a second client on + the same channel never starts a second poll. If the channel is torn down + while the priming is in flight, unsubscribe again — otherwise the poll + would outlive its last subscriber. + """ + from condor.fetchers.portfolio import PORTFOLIO_HISTORY_RANGES + from condor.server_data_service import ServerDataType, get_server_data_service + + sds = get_server_data_service() + + async def _sub(range_key: str) -> None: + try: + await sds.subscribe( + server=server_name, + data_type=ServerDataType.PORTFOLIO_HISTORY, + subscriber_id="ws_manager", + range_key=range_key, + ) + except Exception as e: + logger.debug( + "Failed to subscribe portfolio history %s/%s: %s", + server_name, + range_key, + e, + ) + + await asyncio.gather(*[_sub(rk) for rk in PORTFOLIO_HISTORY_RANGES]) + + if channel not in self._sds_subscriptions: + self._unsubscribe_portfolio_history(server_name) + + def _unsubscribe_portfolio_history(self, server_name: str) -> None: + """Drop this manager's history subscriptions for a server. + + SDS stops polling a key once it has no subscribers left, so this is what + ends the history refresh. + """ + from condor.fetchers.portfolio import PORTFOLIO_HISTORY_RANGES + from condor.server_data_service import ( + CacheKey, + ServerDataType, + get_server_data_service, + ) + + sds = get_server_data_service() + for range_key in PORTFOLIO_HISTORY_RANGES: + sds.unsubscribe( + CacheKey.make( + server_name, + ServerDataType.PORTFOLIO_HISTORY, + range_key=range_key, + ), + "ws_manager", + ) + + # -- SDS listener -- @staticmethod def _transform_executors(raw_data: Any) -> list[dict]: @@ -664,24 +729,15 @@ def _overlay_stopping_state(server_name: str, data: dict) -> None: server_name, data.get("controllers", []), data.get("bots", []) ) - def _on_data_update( - self, server_name: str, cache_key: str, data_type: Any, value: Any - ) -> None: - """Called by SDS when cache is updated. Maps to WS channels and broadcasts.""" - dt_name = data_type.name if hasattr(data_type, "name") else str(data_type) - prefix = _SDT_TO_CHANNEL_PREFIX.get(dt_name) - if not prefix: + def _on_data_update(self, key: CacheKey, value: Any) -> None: + """Called by SDS on every cache write. Maps the key to a WS channel and + broadcasts. ``key`` is an SDS ``CacheKey``: server, data type and params.""" + channel = channel_for_key(key) + if channel is None: return - # Build channel name - if dt_name == "PRICES": - parts = cache_key.split(":") - if len(parts) >= 3: - channel = f"prices:{server_name}:{parts[1]}:{parts[2]}" - else: - return - else: - channel = f"{prefix}:{server_name}" + dt_name = key.data_type.name + server_name = key.server has_subscribers = any(channel in conn.channels for conn in self._connections) if not has_subscribers: @@ -704,7 +760,7 @@ def _on_data_update( logger.debug("Failed to transform bots data for WS: %s", e) return - self._track_oneshot( + self._oneshot_tasks.track( asyncio.create_task( self._broadcast_update(channel, value), name=f"broadcast:{channel}", @@ -1006,10 +1062,10 @@ async def _run_ws_stream( backoff = min(backoff * 2, 60) async def _candle_stream(self, channel: str) -> None: - parts = channel.split(":") - if len(parts) < 5: + parsed = parse_candle_channel(channel) + if parsed is None: return - _, server_name, connector, pair, interval = parts + server_name, connector, pair, interval, pool_address = parsed from config_manager import get_config_manager @@ -1179,10 +1235,10 @@ async def _candle_poll_fallback(self, channel: str) -> None: from — this loop *is* the feed, so it polls from the first tick and never waits for staleness. """ - parts = channel.split(":") - if len(parts) < 5: + parsed = parse_candle_channel(channel) + if parsed is None: return - _, server_name, connector, pair, interval = parts + server_name, connector, pair, interval, pool_address = parsed interval_sec = _INTERVAL_SECONDS.get(interval, 60) gecko = dex_candles.uses_gecko_candles(connector) # How long without a WS update before we consider it stale @@ -1228,7 +1284,7 @@ async def _candle_poll_fallback(self, channel: str) -> None: if gecko: candles = await dex_candles.fetch_dex_candles( connector, - None, + pool_address, pair, interval, now - interval_sec * 5, @@ -1240,27 +1296,17 @@ async def _candle_poll_fallback(self, channel: str) -> None: else: cm = get_config_manager() client = await cm.get_client(server_name) - result = await client.market_data.get_historical_candles( + # No `limit`: this poll only wants the fresh tail of the + # live range, so an empty historical answer stays empty + # rather than pulling a full unrelated window. + candles = await fetch_historical_candles( + client, connector, pair, interval, start_time=now - interval_sec * 5, end_time=now, ) - candles_raw = ( - result - if isinstance(result, list) - else ( - result.get("data", []) - if isinstance(result, dict) - else [] - ) - ) - candles = [ - c - for r in candles_raw - if (c := self._normalize_candle(r)) is not None - ] if candles: buf = self._candle_buffers.get(channel) # Broadcast if we have newer candles OR if the latest @@ -1494,6 +1540,15 @@ async def _executor_stream(self, channel: str) -> None: sds = get_server_data_service() client = await cm.get_client(server_name) all_raw: list[dict] = [] + # Rows already through ExecutorInfo, accumulated page by page. + # Transforming the *whole* accumulated list on every page made + # this quadratic: with FIRST_PAGE/NEXT_PAGE/MAX_PREFETCH below, + # a 5k history cost ~27k pydantic validations instead of 5k, all + # on the event loop the candle and bots broadcasts share. + # ``_transform_executors`` is a pure per-item map over a list + # (``extract_executors_list`` passes a list through untouched), + # so transforming each page once yields the identical snapshot. + transformed: list[dict] = [] page_num = 0 FIRST_PAGE = 50 NEXT_PAGE = 500 @@ -1508,16 +1563,19 @@ async def _executor_stream(self, channel: str) -> None: max_items=MAX_PREFETCH, ): all_raw.extend(page) - - # Transform and broadcast accumulated results after each page - executors = self._transform_executors(all_raw) - if executors: - await self.broadcast(channel, executors) + transformed.extend(self._transform_executors(page)) + + # Broadcast the accumulated snapshot after each page. It is + # copied because ``broadcast`` retains what it is handed as + # the channel's last-known payload, which the next page + # would otherwise keep mutating underneath it. + if transformed: + await self.broadcast(channel, list(transformed)) logger.info( "Executor pre-fetch page %d: %d executors (total %d) for %s", page_num, len(page), - len(executors), + len(transformed), channel, ) page_num += 1 diff --git a/config_manager.py b/config_manager.py index 4182acab6..38b0638c1 100644 --- a/config_manager.py +++ b/config_manager.py @@ -39,6 +39,13 @@ class ServerPermission(str, Enum): ServerPermission.OWNER: 1, } +# The user preference that hands a non-admin seat the in-process code runner +# (SEC-151). It is stored under ``user_preferences[]`` like any other +# preference, but it is a *capability grant*: only an admin may set it, and +# ``condor/web/routes/code.py`` reads it as the fallback arm of its gate. +# Declared here so the gate and the thing that grants it share one spelling. +CODE_RUN_PREFERENCE = "code_run" + class ConfigManager: """ @@ -150,6 +157,7 @@ def _load_config(self): self._data.setdefault("server_access", {}) self._data.setdefault("chat_defaults", {}) self._data.setdefault("user_preferences", {}) + self._data.setdefault("telemetry", {}) # Migrate audit_log from config.yml to separate file (one-time) if "audit_log" in self._data: self._audit_log = self._data.pop("audit_log") @@ -174,6 +182,7 @@ def _default_data(self) -> dict: "server_access": {}, "chat_defaults": {}, "user_preferences": {}, + "telemetry": {}, "version": self.VERSION, } @@ -215,7 +224,12 @@ def _save_config(self): "users": self._data.get("users", {}), "server_access": self._data.get("server_access", {}), "chat_defaults": self._data.get("chat_defaults", {}), + # Omitting this dropped every preference on the next save — + # including the code_run grant, which _load_config reads back + # and routes/code.py gates on (ARCH-177). + "user_preferences": self._data.get("user_preferences", {}), "web_jwt_secret": self._data.get("web_jwt_secret"), + "telemetry": self._data.get("telemetry", {}), "version": self._data.get("version", self.VERSION), } # Keep a copy of the last known-good file before truncating it, @@ -586,6 +600,27 @@ async def close_all_clients(self): # USER MANAGEMENT # ========================================================================= + # ── Telemetry (FEAT-023) ── + # Consent and the install's random identity live here because this file is + # the one durable, process-wide store the MCP subprocess can also read. + # Nothing in this section is transmitted except `install_id` and `level`; + # `install_secret` never leaves the machine. See PRIVACY.md. + + def get_telemetry(self) -> dict: + """The telemetry section. Empty dict on an install that never opted in.""" + section = self._data.get("telemetry") + return dict(section) if isinstance(section, dict) else {} + + def update_telemetry(self, **changes) -> dict: + """Merge keys into the telemetry section and persist.""" + section = self._data.setdefault("telemetry", {}) + if not isinstance(section, dict): + section = {} + self._data["telemetry"] = section + section.update(changes) + self._save_config() + return dict(section) + def get_user(self, user_id: int) -> Optional[dict]: """Get user record.""" return self._data.get("users", {}).get(user_id) @@ -726,6 +761,46 @@ def delete_user_preference(self, user_id: int, key: str) -> bool: return True return False + def has_code_run_grant(self, user_id: int) -> bool: + """Whether this user carries the explicit ``code_run`` grant. + + Says nothing about admins: being an admin is a separate arm of the gate + in ``condor/web/routes/code.py``. This answers only "was the capability + handed to them", which is what an admin UI needs to render. + """ + return bool(self.get_user_preference(user_id, CODE_RUN_PREFERENCE, False)) + + def set_code_run_grant(self, user_id: int, granted: bool, admin_id: int) -> bool: + """Grant or revoke the ``code_run`` capability. Returns True if changed. + + Handing this out is handing out arbitrary Python in the bot's own + process — every server's credentials, the web JWT secret, os.environ — + so it is audited on both edges like ``approve_user``/``block_user``, and + the caller is expected to have already checked ``is_admin(admin_id)``. + + Revoking deletes the key rather than writing ``False``, so config.yml + keeps only the users who actually hold the grant. Either way the write + lands in config.yml immediately and the next request re-reads it, so a + grant takes effect without restarting the bot. + """ + if user_id not in self._data.get("users", {}): + return False + if self.has_code_run_grant(user_id) == bool(granted): + return False + + if granted: + self.set_user_preference(user_id, CODE_RUN_PREFERENCE, True) + else: + self.delete_user_preference(user_id, CODE_RUN_PREFERENCE) + + self._audit( + "code_run_granted" if granted else "code_run_revoked", + "user", + str(user_id), + admin_id, + ) + return True + # ========================================================================= # SERVER ACCESS CONTROL # ========================================================================= diff --git a/frontend/eslint-baseline.json b/frontend/eslint-baseline.json new file mode 100644 index 000000000..ef21e191f --- /dev/null +++ b/frontend/eslint-baseline.json @@ -0,0 +1,44 @@ +{ + "src/components/agent/AgentOverviewTab.tsx | react-hooks/preserve-manual-memoization": 1, + "src/components/agent/AgentPnlChart.tsx | react-refresh/only-export-components": 2, + "src/components/agent/AgentSessionContent.tsx | react-hooks/preserve-manual-memoization": 1, + "src/components/agent/SessionReviewer.tsx | react-hooks/preserve-manual-memoization": 2, + "src/components/bots/AggregatedPnlChart.tsx | react-hooks/set-state-in-render": 1, + "src/components/bots/ControllerBrowser.tsx | react-hooks/set-state-in-effect": 1, + "src/components/bots/ControllerPnlChart.tsx | react-hooks/purity": 1, + "src/components/bots/DeployBotDialog.tsx | react-refresh/only-export-components": 3, + "src/components/chat/ChatInput.tsx | react-hooks/immutability": 1, + "src/components/chat/ChatThread.tsx | react-refresh/only-export-components": 1, + "src/components/chat/DockRoutines.tsx | react-refresh/only-export-components": 1, + "src/components/editor/EditorDialogs.tsx | react-hooks/set-state-in-render": 2, + "src/components/executor/DCAConfigPanel.tsx | @typescript-eslint/no-unused-vars": 1, + "src/components/executor/DCAConfigPanel.tsx | react-refresh/only-export-components": 2, + "src/components/executor/ExecutorTable.tsx | no-empty": 1, + "src/components/executor/fields.tsx | react-hooks/set-state-in-effect": 3, + "src/components/executor/OrderConfigPanel.tsx | react-refresh/only-export-components": 2, + "src/components/executor/PositionConfigPanel.tsx | react-refresh/only-export-components": 2, + "src/components/grid/GridConfigPanel.tsx | react-hooks/set-state-in-effect": 2, + "src/components/grid/GridConfigPanel.tsx | react-refresh/only-export-components": 1, + "src/components/market/MarketsPanel.tsx | react-hooks/purity": 1, + "src/components/market/OrderBook.tsx | react-hooks/immutability": 1, + "src/components/market/PairSelector.tsx | react-hooks/set-state-in-effect": 1, + "src/components/market/PairSelector.tsx | react-refresh/only-export-components": 1, + "src/components/market/PriceTicker.tsx | react-hooks/refs": 3, + "src/components/market/RecentTrades.tsx | react-hooks/set-state-in-effect": 1, + "src/components/routines/ReportBrowser.tsx | react-hooks/set-state-in-effect": 4, + "src/components/routines/RoutineConfigForm.tsx | react-hooks/set-state-in-effect": 1, + "src/components/routines/RoutineHooksPanel.tsx | react-hooks/set-state-in-effect": 1, + "src/components/settings/GatewaySettings.tsx | react-hooks/set-state-in-effect": 1, + "src/components/settings/VoiceSettings.tsx | react-hooks/set-state-in-effect": 1, + "src/hooks/useBacktest.ts | react-hooks/set-state-in-effect": 1, + "src/hooks/useCandleStore.ts | react-hooks/set-state-in-effect": 1, + "src/hooks/useChat.tsx | react-refresh/only-export-components": 2, + "src/pages/BotDetail.tsx | react-hooks/refs": 2, + "src/pages/Bots.tsx | react-hooks/refs": 12, + "src/pages/CreateExecutor.tsx | react-hooks/set-state-in-effect": 1, + "src/pages/Executors.tsx | react-hooks/purity": 1, + "src/pages/Login.tsx | react-hooks/set-state-in-effect": 1, + "src/pages/StrategyDetail.tsx | react-hooks/preserve-manual-memoization": 1, + "src/pages/StrategyDetail.tsx | react-hooks/set-state-in-effect": 1, + "src/pages/tabs/EditorTab.tsx | react-hooks/refs": 6 +} diff --git a/frontend/package.json b/frontend/package.json index e2439edd9..7951deaa4 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -7,6 +7,8 @@ "dev": "vite", "build": "tsc -b && vite build", "lint": "eslint .", + "lint:ci": "node scripts/lint-baseline.mjs", + "lint:baseline": "node scripts/lint-baseline.mjs", "preview": "vite preview" }, "dependencies": { diff --git a/frontend/scripts/lint-baseline.mjs b/frontend/scripts/lint-baseline.mjs new file mode 100644 index 000000000..b76655fe0 --- /dev/null +++ b/frontend/scripts/lint-baseline.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +/** + * Lint gate that fails on *new* errors rather than on the whole backlog. + * + * `eslint .` reports 76 errors that predate the CI gate — mostly React Compiler + * findings (stale refs, cascading setState) that are real but each need their + * own fix. Blocking every PR on all of them would have meant either a + * frontend-wide refactor bolted onto whatever branch got there first, or + * switching the rules off and losing the findings. So the known set is recorded + * in eslint-baseline.json and only what exceeds it fails. + * + * Entries are keyed by file + rule, not by line: line numbers churn on every + * edit above them, which would make the baseline fail for reasons unrelated to + * lint. Counts are compared, so a second violation of an already-known rule in + * an already-known file is still caught. + * + * The baseline only ever ratchets down. A fixed error is reported as drift and + * fails the run too, so the file cannot quietly keep granting an exception that + * is no longer used — rerun with --update to bank the fix. + * + * node scripts/lint-baseline.mjs check (CI) + * node scripts/lint-baseline.mjs --update rewrite the baseline + */ +import { execFileSync } from "node:child_process"; +import { readFileSync, writeFileSync } from "node:fs"; +import { dirname, join } from "node:path"; +import { fileURLToPath } from "node:url"; + +const root = join(dirname(fileURLToPath(import.meta.url)), ".."); +const baselinePath = join(root, "eslint-baseline.json"); +const update = process.argv.includes("--update"); + +function currentCounts() { + let stdout; + try { + // eslint exits non-zero whenever it reports anything, which is the normal + // case here — the report on stdout is what matters, not the status. + stdout = execFileSync("npx", ["eslint", ".", "-f", "json"], { + cwd: root, + encoding: "utf8", + maxBuffer: 64 * 1024 * 1024, + }); + } catch (err) { + if (!err.stdout) throw err; + stdout = err.stdout; + } + + const counts = {}; + for (const file of JSON.parse(stdout)) { + const path = file.filePath.slice(root.length + 1); + for (const msg of file.messages) { + if (msg.severity !== 2) continue; // warnings are not gated + const key = `${path} | ${msg.ruleId ?? "(no-rule)"}`; + counts[key] = (counts[key] ?? 0) + 1; + } + } + return counts; +} + +const counts = currentCounts(); +const total = Object.values(counts).reduce((a, b) => a + b, 0); + +if (update) { + const sorted = Object.fromEntries(Object.entries(counts).sort(([a], [b]) => a.localeCompare(b))); + writeFileSync(baselinePath, `${JSON.stringify(sorted, null, 2)}\n`); + console.log(`Baseline updated: ${Object.keys(sorted).length} entries, ${total} errors.`); + process.exit(0); +} + +let baseline; +try { + baseline = JSON.parse(readFileSync(baselinePath, "utf8")); +} catch { + console.error(`Missing or unreadable ${baselinePath}. Run: npm run lint:baseline -- --update`); + process.exit(1); +} + +const regressions = []; +const fixed = []; +for (const key of new Set([...Object.keys(counts), ...Object.keys(baseline)])) { + const now = counts[key] ?? 0; + const was = baseline[key] ?? 0; + if (now > was) regressions.push(` ${key} (${was} allowed, ${now} found)`); + else if (now < was) fixed.push(` ${key} (${was} allowed, ${now} found)`); +} + +if (regressions.length) { + console.error(`\nNew lint errors not in the baseline:\n${regressions.join("\n")}`); + console.error("\nFix them, or if they are genuinely acceptable, run:"); + console.error(" npm run lint:baseline -- --update\n"); + process.exit(1); +} + +if (fixed.length) { + console.error(`\nBaseline is stale — these are already fixed:\n${fixed.join("\n")}`); + console.error("\nBank the improvement so it cannot regress:"); + console.error(" npm run lint:baseline -- --update\n"); + process.exit(1); +} + +console.log(`No new lint errors. ${total} baselined errors remain to pay down.`); diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index e0051398f..9baf1b4f8 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -11,6 +11,8 @@ import { Agents } from "@/pages/Agents"; import { BotDetail } from "@/pages/BotDetail"; import { Bots } from "@/pages/Bots"; import { CreateExecutor } from "@/pages/CreateExecutor"; +import { Dex } from "@/pages/Dex"; +import { DexPool } from "@/pages/DexPool"; import { Executors } from "@/pages/Executors"; import { Login } from "@/pages/Login"; import { Portfolio } from "@/pages/Portfolio"; @@ -71,6 +73,8 @@ export default function App() { } /> } /> } /> + } /> + } /> } /> } /> } /> diff --git a/frontend/src/components/ConnectKeysOverlay.tsx b/frontend/src/components/ConnectKeysOverlay.tsx index cace8df33..571d23914 100644 --- a/frontend/src/components/ConnectKeysOverlay.tsx +++ b/frontend/src/components/ConnectKeysOverlay.tsx @@ -19,10 +19,11 @@ export function ConnectKeysOverlay() { {/* Copy */}

- Connect Your Exchange + Connect an Exchange or Wallet

- Add your exchange API keys to unlock portfolio tracking, trading, and bot management. + Add exchange API keys or a DEX wallet to unlock portfolio tracking, trading, and bot + management.

{/* CTA */} @@ -30,7 +31,7 @@ export function ConnectKeysOverlay() { onClick={() => navigate("/settings?tab=keys")} className="flex items-center gap-2 rounded-lg bg-[var(--color-primary)] px-6 py-2.5 text-sm font-semibold text-white transition-all hover:bg-[var(--color-primary-hover)] hover:shadow-lg hover:shadow-[var(--color-primary)]/20 active:scale-[0.98]" > - Connect API Keys + Connect Keys & Wallets diff --git a/frontend/src/components/ErrorBoundary.tsx b/frontend/src/components/ErrorBoundary.tsx index 63ca7a2ec..6bbe6fd34 100644 --- a/frontend/src/components/ErrorBoundary.tsx +++ b/frontend/src/components/ErrorBoundary.tsx @@ -1,5 +1,9 @@ import { Component, type ErrorInfo, type ReactNode } from "react"; +import { SERVER_KEY } from "@/lib/auth"; +import { areaForRoute, buildDiagnostics } from "@/lib/diagnostics"; +import { openIssueDraft } from "@/lib/github-issue"; + function isChunkLoadError(error: Error): boolean { const msg = error.message || ""; return ( @@ -18,17 +22,20 @@ interface Props { interface State { hasError: boolean; error: Error | null; + /** React's component stack — the most useful half of a crash report. */ + componentStack: string | null; } export class ErrorBoundary extends Component { - state: State = { hasError: false, error: null }; + state: State = { hasError: false, error: null, componentStack: null }; static getDerivedStateFromError(error: Error): State { - return { hasError: true, error }; + return { hasError: true, error, componentStack: null }; } componentDidCatch(error: Error, info: ErrorInfo) { console.error("[ErrorBoundary]", error, info.componentStack); + this.setState({ componentStack: info.componentStack ?? null }); // Auto-reload once on chunk/module import failures (stale deploys) if (isChunkLoadError(error)) { @@ -43,10 +50,46 @@ export class ErrorBoundary extends Component { componentDidUpdate(prevProps: Props) { if (prevProps.resetKey !== this.props.resetKey && this.state.hasError) { - this.setState({ hasError: false, error: null }); + this.setState({ hasError: false, error: null, componentStack: null }); } } + /** + * File the crash the boundary just caught. + * + * The message and component stack are the report — asking the user to + * retype them is asking for a worse one. Read from `window`/`localStorage` + * rather than router and server context: a class component that renders + * *because* the tree below it failed is the wrong place to add hooks. + */ + private reportCrash = () => { + const error = this.state.error; + const route = window.location.pathname + window.location.search; + const stack = [ + error?.stack || `${error?.name}: ${error?.message}`, + this.state.componentStack ? `\n\nComponent stack:${this.state.componentStack}` : "", + ].join(""); + + openIssueDraft({ + kind: "bug", + title: `Crash: ${(error?.message || "unknown error").slice(0, 80)}`, + area: areaForRoute(route), + description: + "The dashboard crashed and showed the \"Something went wrong\" screen.\n\n" + + `Error: ${error?.message || "unknown"}`, + // The boundary knows the route, not the clicks that led there — the one + // thing it cannot fill in is left for the reporter to write. + detail: "1. \n2. \n3. ", + logs: "```\n" + stack + "\n```", + diagnostics: buildDiagnostics({ + kind: "bug", + route, + server: localStorage.getItem(SERVER_KEY), + env: null, + }), + }); + }; + render() { if (this.state.hasError) { return ( @@ -63,13 +106,21 @@ export class ErrorBoundary extends Component { if (this.state.error && isChunkLoadError(this.state.error)) { window.location.reload(); } else { - this.setState({ hasError: false, error: null }); + this.setState({ hasError: false, error: null, componentStack: null }); } }} className="rounded-md bg-[var(--color-primary)] px-4 py-2 text-sm font-medium text-white transition-colors hover:opacity-90" > {this.state.error && isChunkLoadError(this.state.error) ? "Reload" : "Try Again"} + {this.state.error && !isChunkLoadError(this.state.error) && ( + + )} ); diff --git a/frontend/src/components/ReportIssueDialog.tsx b/frontend/src/components/ReportIssueDialog.tsx new file mode 100644 index 000000000..3a5e66c17 --- /dev/null +++ b/frontend/src/components/ReportIssueDialog.tsx @@ -0,0 +1,250 @@ +import { useQuery } from "@tanstack/react-query"; +import { useState } from "react"; +import { ChevronDown, ChevronRight, ExternalLink } from "lucide-react"; +import { useLocation } from "react-router-dom"; + +import { useEscapeKey } from "@/hooks/useEscapeKey"; +import { useServer } from "@/hooks/useServer"; +import { api } from "@/lib/api"; +import { areaForRoute, buildDiagnostics } from "@/lib/diagnostics"; +import { ISSUE_REPO, openIssueDraft, type IssueKind } from "@/lib/github-issue"; +import { AREAS, type Area } from "@/lib/page-context"; + +/** + * "Report an issue" — writes a GitHub issue draft, does not file it. + * + * The button hands off to GitHub's own form with every field prefilled; the + * user submits it themselves. That keeps the report attributed to them, keeps + * the reply thread in their notifications, and keeps this install free of a + * token that could write to the tracker. + * + * The diagnostics block is opt-in and shown verbatim before it leaves: it ends + * up in a public issue, so the person attaching it gets to read it first. + */ +export function ReportIssueDialog({ + open, + onClose, +}: { + open: boolean; + onClose: () => void; +}) { + const [kind, setKind] = useState("bug"); + const [title, setTitle] = useState(""); + const [description, setDescription] = useState(""); + const [detail, setDetail] = useState(""); + const [area, setArea] = useState(null); + const [includeDiagnostics, setIncludeDiagnostics] = useState(true); + const [showDiagnostics, setShowDiagnostics] = useState(false); + + const { pathname, search } = useLocation(); + const { server } = useServer(); + // Version never changes while the tab is open, and a failed fetch must not + // block the report — the block just says the version is unavailable. + const { data: env } = useQuery({ + queryKey: ["meta-env"], + queryFn: api.getEnv, + staleTime: Infinity, + retry: false, + enabled: open, + }); + + useEscapeKey(open, onClose); + + if (!open) return null; + + const isBug = kind === "bug"; + const route = pathname + search; + // The page the user is on is the best guess at the area, and the best guess + // is what a dropdown should open on — `area` only holds an explicit override. + const effectiveArea = area ?? areaForRoute(route); + const diagnostics = buildDiagnostics({ + kind, + route, + server, + env: env ?? null, + }); + const canSubmit = title.trim().length > 0 && description.trim().length > 0; + + const submit = () => { + if (!canSubmit) return; + openIssueDraft({ + kind, + title, + area: effectiveArea, + description, + detail, + diagnostics: includeDiagnostics ? diagnostics : undefined, + }); + onClose(); + setTitle(""); + setDescription(""); + setDetail(""); + setArea(null); + }; + + const inputClass = + "w-full rounded-md border border-[var(--color-border)] bg-[var(--color-bg)] px-3 py-1.5 text-sm text-[var(--color-text)] focus:border-[var(--color-primary)] focus:outline-none"; + + return ( +
+
e.stopPropagation()} + > +

Report an issue

+

+ Opens a prefilled issue on{" "} + {ISSUE_REPO} in a new tab. You review and + submit it under your own GitHub account — nothing is sent from here. +

+ +
+ {(["bug", "feature"] as const).map((k) => ( + + ))} +
+ +
+
+ + setTitle(e.target.value)} + className={inputClass} + placeholder={ + isBug ? "Portfolio shows a blank chart" : "Filter executors by connector" + } + autoFocus + /> +
+ +
+ + +

+ Preselected from the page you were on. +

+
+ +
+ +