diff --git a/Makefile b/Makefile index a9c1d85c9..d55820f27 100644 --- a/Makefile +++ b/Makefile @@ -69,17 +69,38 @@ check-stopped: exit 1; \ fi +# `remain-on-exit` keeps the pane alive after the process dies, so a startup +# failure can be read back with capture-pane instead of scrolling away with the +# session. Config errors (a bad ADMIN_USER_ID, a mode with no token, a local user +# config.yml has never heard of) exit in the first second and used to leave +# nothing behind but "run make run-fg to see the error". +# +# It is turned back off the moment the boot is confirmed: a crash three hours +# later must still take the session with it, or `make status` and `check-stopped` +# would report a dead Condor as running. run: check-stopped build-frontend @tmux new-session -d -s $(SESSION) -c "$(CURDIR)" 'uv run python main.py' - @sleep 2 - @if tmux has-session -t $(SESSION) 2>/dev/null; then \ - echo "Condor started in tmux session '$(SESSION)'."; \ - echo " make logs - attach (detach: Ctrl+B then D)"; \ - echo " make stop - stop Condor"; \ - else \ + @tmux set-option -t $(SESSION) remain-on-exit on >/dev/null 2>&1 \ + || tmux set-window-option -t $(SESSION) remain-on-exit on >/dev/null 2>&1 || true + @sleep 3 + @if ! tmux has-session -t $(SESSION) 2>/dev/null; then \ echo "Condor exited on startup. Run 'make run-fg' to see the error."; \ exit 1; \ fi + @if [ "$$(tmux list-panes -t $(SESSION) -F '#{pane_dead}' 2>/dev/null | head -1)" = "1" ]; then \ + echo "Condor exited on startup:"; \ + echo ""; \ + tmux capture-pane -p -t $(SESSION) 2>/dev/null | grep -v '^[[:space:]]*$$' | tail -20 | sed 's/^/ /'; \ + echo ""; \ + echo " make run-fg - run in the foreground for the full output"; \ + tmux kill-session -t $(SESSION) >/dev/null 2>&1 || true; \ + exit 1; \ + fi + @tmux set-option -t $(SESSION) remain-on-exit off >/dev/null 2>&1 \ + || tmux set-window-option -t $(SESSION) remain-on-exit off >/dev/null 2>&1 || true + @echo "Condor started in tmux session '$(SESSION)'." + @echo " make logs - attach (detach: Ctrl+B then D)" + @echo " make stop - stop Condor" run-fg: build-frontend uv run python main.py diff --git a/PRIVACY.md b/PRIVACY.md index 3cfc426f3..a2fb1cb2f 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -1,31 +1,33 @@ # Privacy Condor is self-hosted. It runs on your machine, holds your exchange API keys, -and places your orders. So the default for anything that could leave that -machine is **off**, and this document is the complete statement of what the one -optional exception does. - -**Short version:** a fresh install sends nothing. It has no telemetry consent -recorded, which resolves to level `off`, at which the emitter returns before it -has looked at its arguments — no buffer, no file, no directory. Nothing changes -until an admin taps "yes" on a prompt. Even then, this repository has no -collector address compiled into it: with `CONDOR_TELEMETRY_URL` unset, the send -path is inert and events can only ever reach a capped local file. +and places your orders. So what leaves that machine by default is the absolute +minimum — an anonymous "this install exists" — and everything beyond that is +opt-in. This document is the complete statement of both. + +**Short version:** a fresh install counts itself and nothing more. With no +consent recorded it runs at level `ping`: a random install id, the version, and +a periodic heartbeat — nothing about you, your users, or your trading. None of +the usage events are sent until an admin taps "yes" on a prompt. Batches go to +the project's collector at `https://telemetry.hummingbot.org/v1/events`, which +is fixed in the source and cannot be pointed elsewhere. The one full kill +switch is `CONDOR_TELEMETRY=off` in the environment. The whole mechanism is about 900 lines in [`condor/telemetry/`](condor/telemetry/). It is meant to be read, not trusted. --- -## What is collected, if you opt in +## What is collected -There are three levels. You choose one; you can change it later. +There are three levels. The consent prompt chooses between `ping` and `usage`; +`off` exists only as an environment override. You can change the answer later. | Level | What it sends | |---|---| -| `off` | Nothing, ever. The emitter is a no-op. **This is the default.** | -| `ping` | Only that this install exists: `install`, `heartbeat`, `version_change`, `shutdown`. | -| `usage` | The above plus the feature, reliability and agent events below. | +| `ping` | Only that this install exists: `install`, `heartbeat`, `version_change`, `shutdown`. **This is the default, and the floor** — the prompt has no "off" option. | +| `usage` | The above plus the feature, reliability and agent events below. Opt-in only. | +| `off` | Nothing, ever. The emitter is a no-op. Reachable only via `CONDOR_TELEMETRY=off` in the environment. | Every batch carries one context block describing the *deployment*, not you: @@ -35,6 +37,7 @@ Every batch carries one context block describing the *deployment*, not you: | `app.version`, `app.branch` | `54ad4dc`, `main` | Which commit is actually running, so we know what to support. | | `app.python`, `app.os`, `app.arch`, `app.in_docker` | `3.12`, `linux`, `arm64`, `true` | What to test against. | | `config.*` | `has_gateway: true`, `user_count: 3`, `server_count: 2`, `llm_providers: ["openai"]` | Counts and capability flags. Numbers and fixed provider *names* — never a server name, never a URL, never a key. | +| `config.mode` | `telegram` or `local` | Whether the install runs Telegram at all, or only the dashboard. One of two fixed names. | | `dropped` | `0` | How many events the rate limiter discarded. An honest count, so a quiet incident is visibly quiet. | And the events themselves: @@ -94,13 +97,21 @@ per-install, the same person on two installs produces two unrelated hashes. ## Where it goes -Nowhere, unless you configure a destination. +At the default `ping` level, only the four adoption events and the envelope +above. Everything else needs an explicit opt-in. + +Batches are POSTed to `https://telemetry.hummingbot.org/v1/events`. +That address is compiled into +[`condor/telemetry/outbox.py`](condor/telemetry/outbox.py) as `COLLECTOR_URL` +and is not configurable — there is no environment variable or config key that +redirects it, so the only way to change where your data goes is to edit that +line in your own checkout. The collector itself is open source +(`condor-telemetry-server`). -No collector URL is compiled into this repository. Events are buffered in memory -and, when a batch cannot be delivered, appended to +Events are buffered in memory and, when a batch cannot be delivered, appended to `condor/.runtime/telemetry/outbox.jsonl`, which is capped at 5,000 events and 7 -days — oldest dropped. With `CONDOR_TELEMETRY_URL` unset, that is the entire -life cycle: a local file, capped, that you can delete. +days — oldest dropped. At level `off` no event is ever created, so nothing is +buffered, nothing is written, and nothing is sent. You can read exactly what would be sent: @@ -108,33 +119,40 @@ You can read exactly what would be sent: cat condor/.runtime/telemetry/outbox.jsonl | jq . ``` -## How to turn it off — or check it is off +## How to change it — or turn it off entirely -It is already off unless you turned it on. To be certain: +Install counting (`ping`) is the floor: it is not an option on the consent +prompt and cannot be disabled from the dashboard, because the project needs an +honest count of installs to know what to support. To check what your install is +doing: ```bash -# The authoritative answer. Prints "off" on a default install. +# The authoritative answer. Prints "ping" on a default install, +# "off" only when CONDOR_TELEMETRY=off is set. uv run python -c "from condor.telemetry import consent; print(consent.level())" ``` Three ways to control it, in order of precedence: -1. **Environment** — `CONDOR_TELEMETRY=off` in your `.env`. This overrides - everything, in both directions, and is the right answer for a headless or - containerized install. `CONDOR_TELEMETRY_URL` is what enables sending at all; - leave it unset and nothing can be transmitted. -2. **The dashboard API** — `PUT /api/v1/settings/telemetry?level=off` (admin - only). `GET` the same path to see the current state. +1. **Environment** — `CONDOR_TELEMETRY=off` in your `.env`. The one full kill + switch: it overrides everything, suppresses the prompt, and is the right + answer for an install that must send nothing at all. +2. **The dashboard** — Settings → Privacy, which every seat can read and only + the admin can change (`GET`/`PUT /api/v1/settings/telemetry?level=ping|usage`; + `off` is rejected there). This is also where an install that runs without + Telegram is asked in the first place, since it has no bot to be asked + through. 3. **`config.yml`** — edit the `telemetry` section directly: ```yaml telemetry: - consent: denied - level: off + consent: granted + level: ping ``` -Turning it off is a **withdrawal, not a pause**: the in-memory buffer and the -outbox file are deleted, so nothing already recorded can be sent afterwards. +Downgrading from `usage` to `ping` is a **withdrawal, not a pause**: the +in-memory buffer and the outbox file are deleted, so nothing already recorded +can be sent afterwards. ## Changes to this document diff --git a/README.md b/README.md index fac5a7711..1cef972da 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,12 @@ A Telegram bot for monitoring and trading with Hummingbot via the **Hummingbot A > > Full walkthrough: [Securing Condor and Hummingbot API with Tailscale](https://hummingbot.org/blog/posts/securing-condor-and-hummingbot-api-with-tailscale/) · [Hummingbot API Tailscale guide](https://hummingbot.org/hummingbot-api/tailscale/) -> **Privacy:** Condor collects nothing by default. It ships with usage telemetry -> **off** and no collector address, and it stays that way unless an admin -> explicitly opts in from the one prompt it sends on first boot. What that -> option would and would not collect — and how to verify it is off — is spelled -> out in [PRIVACY.md](PRIVACY.md). +> **Privacy:** Condor counts installs anonymously — a random id, the version, +> and a heartbeat; nothing about you or your trading. Everything beyond that is +> **opt-in** from the one prompt it sends on first boot, and +> `CONDOR_TELEMETRY=off` silences telemetry entirely. What each level does and +> does not collect, where it goes, and how to verify it, is spelled out in +> [PRIVACY.md](PRIVACY.md). ## Features @@ -54,7 +55,7 @@ Open Terminal, go to an **empty folder** where you are happy to create files (fo curl -fsSL https://raw.githubusercontent.com/hummingbot/deploy/main/setup.sh | bash ``` -The installer walks you through setup—for example your **Telegram** bot token and your **Telegram user id**—and can also install **Hummingbot API** on the **same machine** if you choose that when it asks. +The installer first asks **how you will use Condor**: **Telegram** (a bot you control from your phone — recommended) or **Local** (no Telegram at all, see [Local mode](#local-mode-no-telegram) below). If you choose Telegram it asks for your bot token and your Telegram user id. Either way it can also install **Hummingbot API** on the **same machine** if you choose that when it asks. When installing Hummingbot API, answer **`y`** when asked to enable Tailscale and paste your auth key. When it finishes, continue to **After installation** below. @@ -96,6 +97,41 @@ The following applies after **Install Condor**. If you used **Install only Hummi - If Condor and the API are on **different machines**, install [Tailscale](https://tailscale.com/download) on the Condor host and add the API in **`/servers`** with host **`hummingbot-api`** (not a public IP). See [Secure Connection via Tailscale](#secure-connection-via-tailscale) below. - If something fails, see **Troubleshooting** below. +## Local mode (no Telegram) + +Answer **2) Local** at the first setup question and Condor runs with **no Telegram bot at all**: no token, no `/web` login link, no polling. `make run` boots the process and the dashboard is at **http://localhost:8088**, already logged in. Everything else is the same product — portfolio, bots, agents, routines, trading — and notifications that would have gone to Telegram land in the dashboard's notification bell instead. + +Use it for development, or to try Condor before deciding whether you want a bot. + +> **Local mode has no login.** Anyone who can reach the port has full trading +> control. That is why it binds **`127.0.0.1` only** — the dashboard is reachable +> from the machine it runs on and nowhere else, not from your LAN and not from +> the internet. +> +> `WEB_HOST=0.0.0.0` in `.env` is the one, deliberate way to expose it. Only set +> it behind something that authenticates for you (Tailscale, an SSH tunnel, an +> authenticating reverse proxy). If you want Condor reachable from your phone, +> the answer is usually Telegram mode, not this. + +The mode is **explicit** and is never guessed from whether a token is present: an +install configured for Telegram whose `TELEGRAM_TOKEN` goes missing **exits at +boot** telling you to run `make setup`. It never falls back to a login-less +dashboard. + +Local mode logs in as `ADMIN_USER_ID` — the same id `config.yml`, your servers, +preferences and defaults already key on. `make setup` writes `ADMIN_USER_ID=1` +for an install that never had a Telegram id; an install that has one **keeps +it**, so switching modes keeps everything you had. + +Switching later is a `make setup` re-run — it shows the current mode and offers +the other one — or editing `CONDOR_MODE` in `.env`. Your `TELEGRAM_TOKEN` is left +in place either way, so switching back is one line. + +A mode that cannot work stops at boot, in `make run`, naming the `.env` line to +fix: telegram mode with no token, an `ADMIN_USER_ID` that is not a positive +integer, or a local user who is not in `config.yml`. None of it is deferred to +the browser. + ## Commands | Command | Description | @@ -233,10 +269,15 @@ Preferences are automatically saved and persist across sessions: ### `.env` ```bash -TELEGRAM_TOKEN=your_bot_token -ADMIN_USER_ID=123456789 +CONDOR_MODE=telegram # telegram (default) or local — see Local mode +TELEGRAM_TOKEN=your_bot_token # Required unless CONDOR_MODE=local +ADMIN_USER_ID=123456789 # Your Telegram user id. Local mode logs in + # as this same id (1 if you never had one) OPENAI_API_KEY=sk-... # Optional, for AI features OPENROUTER_API_KEY=sk-or-... # Optional, unlocks the OpenRouter LLM picker +WEB_HOST=0.0.0.0 # Optional. Overrides the dashboard bind + # address. Local mode binds 127.0.0.1; + # read the Local mode warning first. ``` > **OpenRouter:** Add `OPENROUTER_API_KEY` to `.env`, then in `/agent → Change LLM` @@ -370,6 +411,8 @@ Both `condor` and `hummingbot-api` should appear as connected peers. | Issue | Solution | |-------|----------| | Bot not responding | Check `TELEGRAM_TOKEN` and `ADMIN_USER_ID` in `.env` | +| Exits at boot: `TELEGRAM_TOKEN is not set` | Telegram mode needs a token. Run `make setup` — or choose Local mode there if you do not want a bot | +| Local mode: dashboard not reachable from another device | Intended — local mode binds `127.0.0.1`. See [Local mode](#local-mode-no-telegram) before changing `WEB_HOST` | | Access pending | Admin must approve user via /config > Admin Panel | | Commands failing | Verify Hummingbot API is running | | Connection refused | Check server host:port in `/servers`; use `hummingbot-api` (not `localhost`) when API is on another machine via Tailscale | diff --git a/agents/backpack_mm/AGENT.md b/agents/backpack_mm/AGENT.md deleted file mode 100644 index 79987b40f..000000000 --- a/agents/backpack_mm/AGENT.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -name: Backpack MM -description: Backpack Market Maker Program specialist — volume analysis, market selection, - bot deployment, and ongoing MM operations on Backpack perps -agent_key: claude-acp:sonnet -tools: [] -when_to_consult: When the user wants to analyze Backpack exchange markets, deploy - or manage MM bots on Backpack perps, understand volume requirements for the MM program, - check bot performance against program thresholds, or rotate markets for the Backpack - MM Rewards Program -server_required: true -server_name: local -created_by: 481175164 -created_at: '2026-07-30T06:45:22.685798+00:00' ---- - -# Backpack MM Agent - -You are a specialist in the Backpack Exchange Market Maker Program. You analyze the volume landscape, select markets, deploy and manage pmm_mister bots, and track program qualification metrics. - -## What you do -- Analyze 24h volume per market on Backpack (spot + perpetuals) -- Compute how much maker volume is needed to reach program thresholds -- Assess spread / uptime requirements per market -- Recommend which markets are most tractable given capital constraints -- Deploy and manage pmm_mister bots on Backpack perps -- Monitor bot performance and adjust configs for program compliance -- Rotate markets when volume shifts make qualification harder/easier - -## Program knowledge — Backpack Market Maker Program - -**Reward pool:** $300,000/month — $200k perps, $100k spot - -**Tier score** = max(Total Maker Share, Adjusted Maker Share) -- Total Maker Share = user maker vol / total maker vol (no multipliers) -- Adjusted Maker Share = user adjusted vol / total adjusted vol (lower-liquidity markets get multipliers) - -**Blended Score** (determines reward share per market): -- (Volume Score × 0.8) + (Liquidity Score × 0.2) - -**Volume Score** = user maker vol / total maker vol on that market - -**Liquidity Score** = (Avg Order Size / Avg Spread) × Uptime Score -- Orders must be within **100bps from mid** to count -- Avg order size must be **≥ $2,000** - -**Minimum thresholds to receive rewards on a market:** -- Volume Score ≥ 1.5% -- Avg order size ≥ $2,000 -- At least 1% of total allocation - -**First month:** MM5 maker fees (best tier) regardless of volume. - -## Bot configuration - -Read the `backpack_perp_mm_deploy` skill before any deployment or config change. -The adapted aggressive config uses: -- Connector: `backpack_perpetual` -- **Position mode: ONEWAY** — universal default for all pmm_mister configs on any connector; HEDGE only if explicitly requested. Backpack specifically does not support HEDGE at all (errors on startup) -- 2 levels weighted 2:1, spreads 6bps/15bps -- 300s effectivization, 30s cooldown -- LIMIT_MAKER orders (type 3) for maker credit -- 2x leverage, 5% global SL -- $20K total_amount_quote per controller (min for $2K+ orders) - -## How you answer -- Lead with the key number or recommendation -- Use key: value format, not prose paragraphs -- Always cite the data source and timestamp -- Flag when data is estimated vs fetched live -- If a routine is available, call it and reason over its output — don't guess - -## Routines -- `backpack_volume_scan` — 24h perp volume scan with MM program thresholds diff --git a/agents/backpack_mm/routines/backpack_binance_funding_comparison.py b/agents/backpack_mm/routines/backpack_binance_funding_comparison.py deleted file mode 100644 index b1b9473a9..000000000 --- a/agents/backpack_mm/routines/backpack_binance_funding_comparison.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Compare funding rates between Backpack and Binance perpetuals — interval-normalized carry comparison.""" - -import asyncio -import logging - -import aiohttp -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from condor.reports import ReportBuilder -from routines.base import RoutineResult - -logger = logging.getLogger(__name__) - -CATEGORY = "Arbitrage" - -BACKPACK_API = "https://api.backpack.exchange/api/v1" -BINANCE_FAPI = "https://fapi.binance.com/fapi/v1" - -HOURS_PER_YEAR = 8760 - - -class Config(BaseModel): - """Compare Backpack vs Binance perpetual funding rates, normalized by each exchange's payment interval.""" - - backpack_interval_h: int = Field( - default=1, description="Backpack funding interval in hours (default: 1h)" - ) - binance_interval_h: int = Field( - default=8, description="Binance funding interval in hours (default: 8h)" - ) - min_spread_apr_pct: float = Field( - default=10.0, description="Min annualized spread % to highlight as opportunity" - ) - top_n: int = Field(default=20, description="Max rows in opportunities table") - - -def _normalize_base(base: str) -> str: - """Strip kilo-denomination prefix for cross-exchange matching (1000SHIB / kSHIB → SHIB).""" - if base.startswith("1000"): - return base[4:] - if base and base[0] == "k" and len(base) > 1 and base[1].isupper(): - return base[1:] - return base - - -async def _fetch_backpack_rates( - session: aiohttp.ClientSession, -) -> dict[str, tuple[float, str]]: - """Returns {canonical_base: (rate_per_payment, raw_base)} for all Backpack perp markets.""" - async with session.get(f"{BACKPACK_API}/markets") as resp: - markets = await resp.json() - - perp_symbols = [m["symbol"] for m in markets if m.get("marketType") == "PERP"] - - sem = asyncio.Semaphore(8) - - async def _fetch_one(symbol: str): - async with sem: - try: - async with session.get( - f"{BACKPACK_API}/fundingRates", - params={"symbol": symbol, "limit": 1}, - ) as r: - data = await r.json() - if isinstance(data, list) and data: - rate = data[0].get("fundingRate") - if rate is not None: - return symbol, float(rate) - except Exception as e: - logger.warning(f"Backpack funding fetch failed for {symbol}: {e}") - return symbol, None - - results = await asyncio.gather(*[_fetch_one(s) for s in perp_symbols]) - - out: dict[str, tuple[float, str]] = {} - for symbol, rate in results: - if rate is not None: - raw_base = symbol.replace("_USDC_PERP", "").replace("_USDT_PERP", "") - canonical = _normalize_base(raw_base) - out[canonical] = (rate, raw_base) - - return out - - -async def _fetch_binance_rates(session: aiohttp.ClientSession) -> dict[str, float]: - """Returns {canonical_base: rate_per_payment} for all Binance USDT perp markets.""" - async with session.get(f"{BINANCE_FAPI}/premiumIndex") as resp: - data = await resp.json() - - out: dict[str, float] = {} - for item in data: - symbol = item.get("symbol", "") - if not symbol.endswith("USDT"): - continue - raw_base = symbol[:-4] - canonical = _normalize_base(raw_base) - rate = item.get("lastFundingRate") - if rate is not None: - out[canonical] = float(rate) - - return out - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: - async with aiohttp.ClientSession() as session: - bp_raw, bn_raw = await asyncio.gather( - _fetch_backpack_rates(session), - _fetch_binance_rates(session), - ) - - common = sorted(set(bp_raw) & set(bn_raw)) - - def to_apr(rate_per_payment: float, interval_h: int) -> float: - """Convert a per-payment rate to annualized rate (APR).""" - payments_per_year = HOURS_PER_YEAR / interval_h - return rate_per_payment * payments_per_year - - rows = [] - for base in common: - bp_rate_raw, raw_base = bp_raw[base] - bn_rate_raw = bn_raw[base] - - bp_apr = to_apr(bp_rate_raw, config.backpack_interval_h) - bn_apr = to_apr(bn_rate_raw, config.binance_interval_h) - spread_apr = bp_apr - bn_apr - - # Per-hour rates for a granular comparison column - bp_per_h_bps = (bp_rate_raw / config.backpack_interval_h) * 10000 - bn_per_h_bps = (bn_rate_raw / config.binance_interval_h) * 10000 - spread_per_h_bps = bp_per_h_bps - bn_per_h_bps - - if spread_apr > 0.0001: - trade = "Long BN / Short BP" - elif spread_apr < -0.0001: - trade = "Long BP / Short BN" - else: - trade = "Neutral" - - rows.append( - { - "Asset": base, - "BP Symbol": raw_base, - "BP APR": f"{bp_apr * 100:.2f}%", - "BN APR": f"{bn_apr * 100:.2f}%", - "Spread APR": f"{spread_apr * 100:+.2f}%", - "Spread bps/h": f"{spread_per_h_bps:+.3f}", - "Trade": trade, - "_spread_apr": spread_apr, - "_spread_abs_apr": abs(spread_apr), - "_bp_apr": bp_apr, - "_bn_apr": bn_apr, - "_spread_per_h_bps": spread_per_h_bps, - } - ) - - rows.sort(key=lambda r: r["_spread_abs_apr"], reverse=True) - - min_spread = config.min_spread_apr_pct / 100 - opps = [r for r in rows if r["_spread_abs_apr"] >= min_spread] - top_rows = opps[: config.top_n] - - table_cols = [ - "Asset", - "BP Symbol", - "BP APR", - "BN APR", - "Spread APR", - "Spread bps/h", - "Trade", - ] - clean_top = [{k: r[k] for k in table_cols} for r in top_rows] - clean_all = [{k: r[k] for k in table_cols} for r in rows[:60]] - - # ── Report ───────────────────────────────────────────────────────────── - builder = ReportBuilder("Backpack vs Binance Funding (Interval-Normalized)") - builder.source("routine", "backpack_binance_funding_comparison") - builder.tags(["funding", "arbitrage", "backpack", "binance", "carry"]) - builder.manual_order() - - builder.section("01 / OVERVIEW") - builder.kpi("Backpack Perps", str(len(bp_raw))) - builder.kpi("Binance Perps", str(len(bn_raw))) - builder.kpi("Common Markets", str(len(common))) - builder.kpi(f"Opps ≥{config.min_spread_apr_pct}% APR", str(len(opps))) - builder.kpi("BP Interval", f"{config.backpack_interval_h}h") - builder.kpi("BN Interval", f"{config.binance_interval_h}h") - - builder.markdown( - f"Rates are **annualized**: Backpack raw rate ÷ {config.backpack_interval_h}h × 8,760 h/yr; " - f"Binance raw rate ÷ {config.binance_interval_h}h × 8,760 h/yr. " - f"**Spread bps/h** = (BP per-hour rate − BN per-hour rate) × 10,000 — " - f"the carry you earn or pay per hour of holding the delta-neutral pair." - ) - - dataset_rows = [ - { - "asset": r["Asset"], - "bp_apr_pct": round(r["_bp_apr"] * 100, 2), - "bn_apr_pct": round(r["_bn_apr"] * 100, 2), - "spread_apr_pct": round(r["_spread_apr"] * 100, 2), - "abs_spread_apr_pct": round(r["_spread_abs_apr"] * 100, 2), - "spread_bps_per_h": round(r["_spread_per_h_bps"], 3), - } - for r in rows - ] - builder.dataset("funding", dataset_rows) - builder.chart( - "bar", - "Spread (APR %) by Asset", - "funding", - "asset", - "spread_apr_pct", - y_label="Spread APR (%)", - x_label="Asset", - reference_lines=[{"axis": "y", "value": 0, "label": "Zero"}], - ) - - builder.section( - "02 / OPPORTUNITIES", - f"Sorted by |spread APR|. Threshold: {config.min_spread_apr_pct}% APR. " - f"BP pays every {config.backpack_interval_h}h, BN every {config.binance_interval_h}h.", - ) - if clean_top: - builder.table(clean_top, table_cols) - else: - builder.markdown( - f"No common markets with |spread| ≥ {config.min_spread_apr_pct}% APR." - ) - - builder.section("03 / ALL COMMON MARKETS", "Full list sorted by |spread APR|.") - builder.table(clean_all, table_cols) - - report_id = await builder.save() - - # ── Inline summary ────────────────────────────────────────────────────── - top3 = top_rows[:3] - lines = [ - f"Backpack vs Binance Funding (BP={config.backpack_interval_h}h / BN={config.binance_interval_h}h intervals)", - f"{len(common)} common markets | {len(opps)} opps ≥{config.min_spread_apr_pct}% APR", - "", - ] - if top3: - for r in top3: - lines.append( - f" {r['Asset']:8s} BP={r['BP APR']:>9s} BN={r['BN APR']:>9s}" - f" spread={r['Spread APR']:>9s} → {r['Trade']}" - ) - else: - lines.append(" No opportunities above threshold.") - lines.append(f"\nReport: {report_id}") - - return RoutineResult( - text="\n".join(lines), - table_data=clean_top[:10], - table_columns=table_cols, - ) diff --git a/agents/backpack_mm/routines/backpack_funding_compare.py b/agents/backpack_mm/routines/backpack_funding_compare.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/agents/backpack_mm/routines/backpack_mm_volume_analysis.py b/agents/backpack_mm/routines/backpack_mm_volume_analysis.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/agents/backpack_mm/routines/backpack_volume_scan.py b/agents/backpack_mm/routines/backpack_volume_scan.py deleted file mode 100644 index 5d7373437..000000000 --- a/agents/backpack_mm/routines/backpack_volume_scan.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Scan Backpack Exchange perp markets: 24h volume, MM program thresholds, and tractability ranking.""" - -import logging - -import aiohttp -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from condor.reports import ReportBuilder -from routines.base import RoutineResult - -logger = logging.getLogger(__name__) - -CATEGORY = "Market Data" - -BACKPACK_API = "https://api.backpack.exchange/api/v1" - - -class Config(BaseModel): - """Backpack perp volume scan — MM program threshold analysis""" - - target_share_pct: float = Field( - default=0.5, description="Target maker volume share % (e.g. 0.5 = 0.5%)" - ) - min_volume_score_pct: float = Field( - default=1.5, description="Min Volume Score % to qualify on a market" - ) - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: - async with aiohttp.ClientSession() as session: - # Fetch tickers and markets in parallel - async with session.get(f"{BACKPACK_API}/tickers") as resp: - if resp.status != 200: - return f"Backpack API error: {resp.status}" - tickers = await resp.json() - - async with session.get(f"{BACKPACK_API}/markets") as resp: - if resp.status != 200: - return f"Backpack markets API error: {resp.status}" - markets = await resp.json() - - # Build market info lookup - market_info = {m["symbol"]: m for m in markets if m.get("marketType") == "PERP"} - - # Filter perp tickers and parse volumes - perps = [] - for t in tickers: - sym = t.get("symbol", "") - if not sym.endswith("_PERP"): - continue - quote_vol = float(t.get("quoteVolume", 0)) - last_price = float(t.get("lastPrice", 0)) - trades = int(t.get("trades", 0)) - price_change_pct = float(t.get("priceChangePercent", 0)) * 100 - base = sym.replace("_USDC_PERP", "").replace("k", "") - - # Get margin requirements from market info - info = market_info.get(sym, {}) - imf_base = float(info.get("imfFunction", {}).get("base", 0)) - max_leverage = round(1 / imf_base, 1) if imf_base > 0 else 0 - - perps.append( - { - "symbol": sym, - "base": base, - "quote_volume_24h": quote_vol, - "last_price": last_price, - "trades_24h": trades, - "price_change_pct": round(price_change_pct, 2), - "max_leverage": max_leverage, - } - ) - - if not perps: - return "No perpetual markets found on Backpack" - - # Sort by volume descending - perps.sort(key=lambda x: x["quote_volume_24h"], reverse=True) - - total_volume = sum(p["quote_volume_24h"] for p in perps) - - # Calculate thresholds - target_share = config.target_share_pct / 100 - min_vol_score = config.min_volume_score_pct / 100 - - target_total_volume = total_volume * target_share # 0.5% of total perp volume - monthly_target = target_total_volume * 30 - - # Per-market analysis - table_data = [] - for p in perps: - vol = p["quote_volume_24h"] - vol_share = (vol / total_volume * 100) if total_volume > 0 else 0 - # Volume needed for 1.5% Volume Score on this market - needed_for_qual = vol * min_vol_score / (1 - min_vol_score) - # Volume needed for target share on this market - needed_for_target = vol * target_share / (1 - target_share) - - table_data.append( - { - "Market": p["base"], - "24h Vol": f"${vol:,.0f}", - "Vol Share": f"{vol_share:.1f}%", - "Trades": f"{p['trades_24h']:,}", - "Price": ( - f"${p['last_price']:,.4f}" - if p["last_price"] < 1 - else f"${p['last_price']:,.2f}" - ), - "24h Chg": f"{p['price_change_pct']:+.1f}%", - "Max Lev": f"{p['max_leverage']:.0f}x", - "Need 1.5% VS": f"${needed_for_qual:,.0f}", - "Need 0.5% Total": f"${needed_for_target:,.0f}", - "_vol": vol, - "_needed_qual": needed_for_qual, - "_needed_target": needed_for_target, - } - ) - - # Build report - builder = ReportBuilder("Backpack Perp Volume Scan") - builder.source("routine", "backpack_volume_scan") - builder.tags(["backpack", "volume", "market-making", "perps"]) - builder.manual_order() - - builder.section("01 / PROGRAM OVERVIEW") - builder.kpi("Total Perp Volume (24h)", f"${total_volume:,.0f}") - builder.kpi("Perp Markets", str(len(perps))) - builder.kpi( - f"Target Share ({config.target_share_pct}%)", f"${target_total_volume:,.0f}/day" - ) - builder.kpi("Monthly Target (30d)", f"${monthly_target:,.0f}") - builder.kpi("Min Vol Score Threshold", f"{config.min_volume_score_pct}%") - - builder.section("02 / VOLUME BY MARKET") - builder.markdown( - "**Need 1.5% VS** = maker volume needed on that market to hit the 1.5% Volume Score minimum.\n\n" - f"**Need {config.target_share_pct}% Total** = maker volume needed on that market alone to reach " - f"{config.target_share_pct}% of total perp maker volume." - ) - - # Interactive dataset - dataset_rows = [] - for p in perps: - vol = p["quote_volume_24h"] - vol_share = (vol / total_volume * 100) if total_volume > 0 else 0 - needed_qual = vol * min_vol_score / (1 - min_vol_score) - needed_target = vol * target_share / (1 - target_share) - dataset_rows.append( - { - "market": p["base"], - "volume_24h": round(vol, 2), - "vol_share_pct": round(vol_share, 2), - "trades_24h": p["trades_24h"], - "last_price": p["last_price"], - "change_pct": p["price_change_pct"], - "max_leverage": p["max_leverage"], - "need_1_5_pct_vs": round(needed_qual, 2), - "need_target_share": round(needed_target, 2), - } - ) - - builder.dataset("perps", dataset_rows) - - builder.chart( - "bar", - "24h Volume by Market (USD)", - "perps", - "market", - "volume_24h", - y_label="Volume (USD)", - x_label="Market", - ) - - # Display table (clean columns without internal fields) - display_cols = [ - "Market", - "24h Vol", - "Vol Share", - "Trades", - "Price", - "24h Chg", - "Max Lev", - "Need 1.5% VS", - f"Need {config.target_share_pct}% Total", - ] - clean_table = [{k: row[k] for k in display_cols} for row in table_data] - builder.table(clean_table, display_cols) - - builder.section("03 / TRACTABILITY RANKING") - builder.markdown( - "Markets sorted by **lowest maker volume needed** to hit 1.5% Volume Score — " - "these are the easiest to qualify on." - ) - - # Sort by needed_for_qual ascending (most tractable first) - tractable = sorted(table_data, key=lambda x: x["_needed_qual"]) - tract_table = [] - for i, row in enumerate(tractable[:15], 1): - tract_table.append( - { - "#": i, - "Market": row["Market"], - "24h Vol": row["24h Vol"], - "Need for 1.5% VS": row["Need 1.5% VS"], - "Daily @ Target": row[f"Need {config.target_share_pct}% Total"], - } - ) - builder.table( - tract_table, ["#", "Market", "24h Vol", "Need for 1.5% VS", "Daily @ Target"] - ) - - builder.section("04 / STRATEGY NOTES") - builder.markdown( - f"**Reward pool**: $200k/month for perps\n\n" - f"**Blended Score** = (Volume Score × 0.8) + (Liquidity Score × 0.2)\n\n" - f"**Volume Score** = your maker vol / total maker vol on that market\n\n" - f"**Liquidity Score** = (Avg Order Size / Avg Spread) × Uptime\n\n" - f"**Minimum thresholds per market**: Volume Score ≥ 1.5%, Avg order ≥ $2,000\n\n" - f"**Tier Score** = max(Total Maker Share, Adjusted Maker Share) — " - f"lower-liquidity markets get multipliers in the adjusted calculation, " - f"making them more rewarding per dollar traded.\n\n" - f"**First month**: MM5 maker fees (best tier) regardless of volume." - ) - - report_id = await builder.save() - - # Build inline summary - top5 = perps[:5] - top5_lines = "\n".join( - f" {p['base']}: ${p['quote_volume_24h']:,.0f} (need ${p['quote_volume_24h'] * min_vol_score / (1 - min_vol_score):,.0f} for 1.5% VS)" - for p in top5 - ) - easiest3 = sorted(perps, key=lambda x: x["quote_volume_24h"])[:3] - easiest_lines = "\n".join( - f" {p['base']}: need only ${p['quote_volume_24h'] * min_vol_score / (1 - min_vol_score):,.0f}/day" - for p in easiest3 - ) - - summary = ( - f"Backpack Perp Volume Scan\n" - f"Total 24h perp volume: ${total_volume:,.0f}\n" - f"Target ({config.target_share_pct}% share): ${target_total_volume:,.0f}/day\n\n" - f"Top 5 by volume:\n{top5_lines}\n\n" - f"Easiest to qualify (1.5% VS):\n{easiest_lines}\n\n" - f"Report: {report_id}" - ) - - return RoutineResult( - text=summary, - table_data=[ - {k: row[k] for k in ["Market", "24h Vol", "Vol Share", "Need 1.5% VS"]} - for row in table_data[:10] - ], - table_columns=["Market", "24h Vol", "Vol Share", "Need 1.5% VS"], - ) diff --git a/agents/backpack_mm/skills/backpack_perp_mm_deploy/SKILL.md b/agents/backpack_mm/skills/backpack_perp_mm_deploy/SKILL.md deleted file mode 100644 index 71f6619c8..000000000 --- a/agents/backpack_mm/skills/backpack_perp_mm_deploy/SKILL.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: backpack_perp_mm_deploy -description: End-to-end playbook for deploying pmm_mister bots on Backpack perps optimized - for the MM Rewards Program — adapted aggressive config, volume targets, and program - compliance. -when_to_use: When deploying, configuring, or tuning market making bots on Backpack - perpetuals. Also when asked about Backpack MM program compliance, order sizing, - or spread settings for the rewards program. -created: '2026-08-10T21:14:30Z' -source: agent:backpack_mm ---- - -# Backpack Perp MM Deploy Playbook - -Adapted from the market_making_expert's aggressive pmm_mister profile, -tuned specifically for the **Backpack MM Rewards Program** on perpetuals. - -## Program constraints (hard requirements) - -- **Avg order size ≥ $2,000** — orders below this don't count -- **Orders within 100bps from mid** — spreads must be < 0.01 -- **Volume Score ≥ 1.5%** per market to receive rewards -- **Post-only orders** — must be LIMIT_MAKER (type 3) for maker volume -- **Uptime matters** — Liquidity Score = (Avg Order Size / Avg Spread) × Uptime - -## Config template — Backpack MM Aggressive - -Fetch the full config from the companion file: -``` -manage_skill(action="read_file", name="backpack_perp_mm_deploy", file="config_backpack_aggressive.md") -``` - -### Key adaptations vs standard aggressive profile - -| Parameter | Standard Aggressive | Backpack MM | Why | -|-----------|-------------------|-------------|-----| -| total_amount_quote | $500 | **$20,000** | Min $2K orders need larger capital | -| levels | 2 equal | **2 weighted (2:1)** | Front-load L1 for volume, L2 for depth/Liquidity Score | -| spreads | 8bps, 15bps | **6bps, 15bps** | Tighter L1 for fills, wide L2 for depth | -| effectivization_time | 60s | **300s** | More time for TP to fill on thin markets → volume cycling | -| take_profit | 8bps | **6bps** | Fast cycle, viable with MM5 fees (first month) | -| cooldown_time | 30s | **30s** | Prevent adverse stacking on thin markets | -| leverage | 10x | **2x** | Mid-tier perps are volatile, protect capital | -| order_type | 3 | **3** | LIMIT_MAKER mandatory for program credit | - -## Deployment steps - -1. **Run volume scan** — `manage_routines(action="run", name="backpack_volume_scan")` - to get fresh 24h volume data and identify tractable markets. - -2. **Pick markets** — Select 3-5 mid-tier perps where needed maker volume for - 1.5% VS is achievable ($8-15K/day range). Avoid BTC (needs $1M+/day). - -3. **Verify pairs exist on Backpack perps** — Before creating any config, confirm - each ticker actually exists as a perpetual on Backpack by checking - `GET https://api.backpack.exchange/api/v1/markets`. The volume scan may report - spot-style tickers that don't match the actual perp symbol. Known gotchas: - - BONK perp = **kBONK**, SHIB perp = **kSHIB** (kilo-denominated) - - TRX, STABLE — **no perp exists** - - When in doubt, fetch the markets list and grep for the symbol - -4. **Read config template** — fetch `config_backpack_aggressive.md` companion file. - -5. **Adapt per market** — substitute `trading_pair`, verify leverage is within - the market's max (check `imfFunction.base` from /api/v1/markets). - -6. **Set position mode** — Backpack perpetual **only supports ONEWAY** position - mode. Always set `"position_mode": "ONEWAY"`. Never use `HEDGE` — the bot - will error on startup if HEDGE is set. - -7. **Upsert configs** — one config per market via `manage_controllers(action="upsert", target="config", ...)`. - Name convention: `NNN_pmm_backpack_`. - -8. **Deploy single bot** with all controllers: - ``` - manage_bots(action="deploy", bot_name="backpack_mm_perps_", - controllers_config=["config1", "config2", "config3"]) - ``` - -9. **Verify** — `manage_bots(action="status")` to confirm all controllers running. - -## Volume math (per controller) - -- Order size L1: $20K × 0.25 × (2/3) = **$3,333** ✓ (above $2K) -- Order size L2: $20K × 0.25 × (1/3) = **$1,667** (depth, may not count for avg) -- Per complete fill→TP cycle: 2 × $3,333 = **$6,666** maker volume -- Need ~2-3 cycles/day per market for 1.5% VS on $900K market -- At 300s effectivization + 30s cooldown: achievable - -## Monitoring - -- Schedule `backpack_volume_scan` daily to track volume shifts -- Check bot PnL and fill rates via mm_bot_report -- Watch for markets dropping below threshold — may need to rotate diff --git a/agents/backpack_mm/skills/backpack_perp_mm_deploy/config_backpack_aggressive.md b/agents/backpack_mm/skills/backpack_perp_mm_deploy/config_backpack_aggressive.md deleted file mode 100644 index 9f7790b19..000000000 --- a/agents/backpack_mm/skills/backpack_perp_mm_deploy/config_backpack_aggressive.md +++ /dev/null @@ -1,94 +0,0 @@ -# Backpack MM Aggressive Config Template - -Adapted from the MM expert's aggressive profile, tuned for Backpack MM Rewards -Program compliance. All orders are LIMIT_MAKER (post-only) for maker credit. - -## Config (substitute connector/pair/amount per market) - -```json -{ - "controller_type": "generic", - "controller_name": "pmm_mister", - "connector_name": "backpack_perpetual", - "trading_pair": "-USDC", - "total_amount_quote": 20000, - "portfolio_allocation": 0.25, - "target_base_pct": 0.5, - "min_base_pct": 0.3, - "max_base_pct": 0.7, - "buy_spreads": "0.0003,0.0015", - "sell_spreads": "0.0003,0.0015", - "buy_amounts_pct": "2,1", - "sell_amounts_pct": "2,1", - "executor_refresh_time": 20, - "buy_cooldown_time": 30, - "sell_cooldown_time": 30, - "buy_position_effectivization_time": 300, - "sell_position_effectivization_time": 300, - "price_distance_tolerance": 0.0005, - "refresh_tolerance": 0.0003, - "tolerance_scaling": 1.1, - "open_order_type": 3, - "take_profit": 0.0006, - "take_profit_order_type": 3, - "leverage": 2, - "position_mode": "ONEWAY", - "position_side": "BUY", - "max_active_executors_by_level": 2, - "tick_mode": false, - "min_skew": 1.0, - "global_tp_enabled": false, - "global_sl_enabled": true, - "global_stop_loss": 0.05, - "global_sl_activation_from": "target_base", - "global_pnl_reference": "position" -} -``` - -## Position mode — universal default - -**Always use `"position_mode": "ONEWAY"`** for pmm_mister configs on any connector. -This is the universal default — not just a Backpack restriction. HEDGE should only -be used when the user explicitly requests it. - -On Backpack specifically, HEDGE is not supported at all and will error on startup. -On other connectors (Hyperliquid, Binance, etc.), ONEWAY is still the default unless -the user says otherwise. - -## Parameter rationale - -**Order sizing** (meets $2K minimum): -- Level 1: $20K × 0.25 × (2/3) = $3,333 per order -- Level 2: $20K × 0.25 × (1/3) = $1,667 per order (depth) -- Both sides quoting = continuous presence - -**Spreads** (within 100bps program limit): -- L1: 3bps — very tight, maximizes fill rate and volume generation -- L2: 15bps — passive depth, helps Liquidity Score - -**Effectivization** (300s): -- After a fill, the LIMIT_MAKER TP order stays on book for 5 minutes -- On mid-tier perps, price needs time to cycle back for TP fill -- Complete fill→TP cycle = 2× notional in maker volume -- 60s was too short (most positions expired to hold); 300s catches ~50%+ of TPs - -**Take profit** (6bps): -- With MM5 fees (~0.02% maker), round-trip ~0.04% -- 6bps TP gives 1.5× fee coverage — tight but viable -- If Backpack fees are higher than expected, bump to 8bps - -**Leverage** (2x): -- Mid-tier perps (ZEC, PUMP, WLD) can gap 5-10% -- At 2x: 5% move = 10% drawdown → survivable with 5% global SL -- At 5x: 5% move = 25% drawdown → dangerous - -**Cooldown** (30s): -- Prevents stacking fills at similar prices on thin markets -- 15s was too aggressive for $500-900K/day volume markets - -## Adjustments per market - -- **PAXG**: Gold-backed, low vol → can keep 3bps, reduce cooldown to 20s -- **ZEC/WLD**: Moderate vol → use template as-is -- **PUMP**: Higher vol, can gap → monitor closely with 3bps, widen to 5bps if adverse fills spike -- **Leverage**: Always verify max leverage per market (1/imfFunction.base from API) diff --git a/agents/backpack_mm/strategies/backpack_perp_mm/strategy.md b/agents/backpack_mm/strategies/backpack_perp_mm/strategy.md deleted file mode 100644 index 35c9c1b0c..000000000 --- a/agents/backpack_mm/strategies/backpack_perp_mm/strategy.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -name: Backpack Perp MM -description: '' -agent_key: null -skills: [] -default_config: - frequency_sec: 3600 - total_amount_quote: 60000 -default_trading_context: '' -created_by: 481175164 -created_at: '2026-08-10T21:16:55.078013+00:00' ---- - -# Backpack Perp MM Strategy - -Market making on Backpack perpetuals targeting the MM Rewards Program. -Agent pinned to `local` server. - -## Loop playbook - -Each tick: - -1. **Volume check** — Run `backpack_volume_scan` routine to get fresh 24h volumes. - Compare current market volumes against previous tick. Flag markets where volume - dropped below qualification threshold. - -2. **Bot health** — Check `manage_bots(action="status")` for the running bot. - Verify all controllers are in `running` state. Check for errors in logs. - -3. **Performance assessment** — For each controller: - - Current PnL (realized + unrealized) - - Volume generated vs target (need 1.5% VS per market) - - Inventory balance (base_pct within min/max band) - - Fill rate and TP hit rate - -4. **Compliance check** — Verify program metrics: - - Are avg order sizes ≥ $2,000? - - Are spreads within 100bps? - - Is uptime acceptable for Liquidity Score? - -5. **Adjust if needed**: - - If a market's volume dropped and qualification is at risk → consider rotating - to a more tractable market - - If inventory is drifting → adjust target/min/max_base_pct live - - If PnL is bleeding → widen spreads or increase cooldown - - If volume target is not being met → tighten spreads cautiously - -6. **Journal** — Write a summary of the tick: volumes, PnL, compliance status, - any adjustments made. - -## Active deployment - -- Bot: `backpack_mm_perps_20260811-20260810-211539` -- Controllers (all 3bps L1 / 15bps L2, 6bps TP, 2x leverage, $20K each): - - `001_pmm_backpack_ZEC-USDC` — ZEC perp - - `002_pmm_backpack_WLD-USDC` — WLD perp - - `003_pmm_backpack_PUMP-USDC` — PUMP perp - -## Risk limits - -- Max $20K per controller ($60K total across 3 markets) -- 2x leverage -- 5% global stop loss per controller -- No global TP — let volume cycle - -## Skills to read - -- `backpack_perp_mm_deploy` — full config template and deployment steps -- `backpack_volume_scan` routine — fresh volume data diff --git a/agents/brigado/AGENT.md b/agents/brigado/AGENT.md deleted file mode 100644 index b81b6a9d5..000000000 --- a/agents/brigado/AGENT.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -name: Brigado -description: BRL market making research agent. Tracks USDT-BRL spreads, BTC/fiat - volume dynamics, and bot performance for the Brazilian market making operation. -agent_key: claude-acp:sonnet -tools: [] -when_to_consult: '' -server_required: true -created_by: 481175164 -created_at: '2026-05-04T00:00:00+00:00' ---- - -## Brigado - -### Objective -Research and monitoring agent for the Brazilian market making operation. Provides dashboards, volume analysis, and bot performance tracking focused on BRL pairs and fiat market dynamics. diff --git a/agents/brigado/routines/bot_report.py b/agents/brigado/routines/bot_report.py deleted file mode 100644 index 561257bea..000000000 --- a/agents/brigado/routines/bot_report.py +++ /dev/null @@ -1,1752 +0,0 @@ -"""Bot Performance Report — Clean 2-message report with USD values and rebate tracking.""" - -CATEGORY = "Bot Analysis" - -import asyncio -import logging -import os -import re -from collections import defaultdict -from datetime import datetime, timezone - -import plotly.graph_objects as go -from plotly.subplots import make_subplots -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from config_manager import get_client, get_config_manager - -logger = logging.getLogger(__name__) - -REBATE_RATE = 0.00015 # 0.015% of volume — only applies to BRL-quoted pairs - -# Config analysis constants -_CONFIG_SKIP_PARAMS = { - "id", - "controller_type", - "controller_name", - "manual_kill_switch", - "_config_name", -} -_CONFIG_KEY_PARAMS = { - "buy_spreads", - "sell_spreads", - "buy_amounts_pct", - "sell_amounts_pct", - "total_amount_quote", - "stop_loss", - "take_profit", - "time_limit", - "executor_refresh_time", - "cooldown_time", - "leverage", - "dca_spreads", - "dca_amounts", - "trailing_stop", - "executor_activation_bounds", - "top_executor_refresh_time", - "position_rebalance_threshold_pct", -} - - -def _pair_has_rebate(pair: str, quote_currency: str) -> bool: - """Rebate only applies to pairs quoted in the rebate currency (BRL).""" - if not pair or "-" not in pair: - return False - return pair.split("-")[-1].upper() == quote_currency.upper() - - -class Config(BaseModel): - """Generate a formatted performance report for all active bots.""" - - quote_currency: str = Field( - default="BRL", - description="Quote currency of the bots (for USD conversion)", - ) - target_chat_id: int | None = Field( - default=None, - description="Send report to this chat ID instead of the current chat (e.g. a group chat)", - ) - server_name: str = Field( - default="brigado_2", - description="Server to connect to for bot data (bots run on this server)", - ) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _fmt(n: float, decimals: int = 2) -> str: - if abs(n) >= 1_000: - return f"{n:,.{decimals}f}" - return f"{n:.{decimals}f}" - - -def _sign(n: float) -> str: - return "+" if n >= 0 else "-" - - -def _emoji(n: float) -> str: - return "🟢" if n > 0 else "🔴" if n < 0 else "⚪" - - -def _md_escape(text: str) -> str: - """Escape special characters for Telegram MarkdownV2.""" - special = r"_*[]()~`>#+-=|{}.!" - result = [] - for ch in text: - if ch in special: - result.append(f"\\{ch}") - else: - result.append(ch) - return "".join(result) - - -# --------------------------------------------------------------------------- -# Data fetching -# --------------------------------------------------------------------------- - - -async def _get_bots_data(client) -> dict: - data = await client.bot_orchestration.get_active_bots_status() - if isinstance(data, dict) and "data" in data: - return data["data"] - return {} - - -async def _get_bot_runs(client) -> dict: - runs = {} - try: - data = await client.bot_orchestration.get_bot_runs() - if isinstance(data, dict) and "data" in data: - for run in data["data"]: - if run.get("deployment_status") == "DEPLOYED" and run.get( - "deployed_at" - ): - runs[run["bot_name"]] = run["deployed_at"] - except Exception: - pass - return runs - - -async def _get_prices(client, connector: str, pairs: list[str]) -> dict[str, float]: - try: - result = await client.market_data.get_prices( - connector_name=connector, trading_pairs=",".join(pairs) - ) - return result.get("prices", {}) if result else {} - except Exception as e: - logger.warning(f"Failed to get prices: {e}") - return {} - - -async def _get_balances(client) -> dict: - """Fetch portfolio balances from all accounts.""" - try: - return await client.portfolio.get_state(refresh=False) - except Exception as e: - logger.warning(f"Failed to get balances: {e}") - return {} - - -async def _get_market_volume_since( - client, connector: str, pair: str, since_dt: datetime | None -) -> tuple[float, float]: - """Return (base_volume, quote_volume) accumulated from candles since *since_dt*. - - Uses 1h candles spanning from *since_dt* to now so the market volume window - matches the bot's runtime, giving an accurate market-share calculation. - Falls back to a single 1d candle when *since_dt* is unavailable. - """ - try: - if since_dt: - now = datetime.now(timezone.utc) - days = max(1, int((now - since_dt).total_seconds() / 86400) + 1) - candles = await client.market_data.get_candles_last_days( - connector_name=connector, - trading_pair=pair, - days=days, - interval="1h", - ) - else: - candles = await client.market_data.get_candles( - connector_name=connector, - trading_pair=pair, - interval="1d", - max_records=1, - ) - - if candles: - data = candles if isinstance(candles, list) else candles.get("data", []) - if not data: - return 0.0, 0.0 - - since_ts = since_dt.timestamp() if since_dt else 0 - total_base = 0.0 - total_quote = 0.0 - for row in data: - # Only count candles that start at or after the deploy time - ts = float(row.get("timestamp", 0) or 0) - # Normalise ms → s if needed - if ts > 1e12: - ts /= 1000 - if ts >= since_ts: - total_base += float(row.get("volume", 0) or 0) - total_quote += float(row.get("quote_asset_volume", 0) or 0) - return total_base, total_quote - except Exception as e: - logger.warning(f"Failed to get volume for {pair}: {e}") - return 0.0, 0.0 - - -# --------------------------------------------------------------------------- -# Chart builders (Plotly) -# --------------------------------------------------------------------------- - -_DARK_BG = "#1a1a2e" -_DARK_PAPER = "#16213e" -_GREEN = "#00d4aa" -_RED = "#ff6b6b" -_BLUE = "#58a6ff" -_YELLOW = "#ffdd57" -_COLORS = [ - "#58a6ff", - "#00d4aa", - "#ff6b6b", - "#ffdd57", - "#c084fc", - "#fb923c", - "#38bdf8", - "#f472b6", -] - - -def _chart_layout(title: str, **kwargs) -> dict: - base = dict( - title=dict(text=title, font=dict(color="#e6edf3", size=14), x=0.5), - paper_bgcolor=_DARK_PAPER, - plot_bgcolor=_DARK_BG, - font=dict(color="#ccc", size=11), - margin=dict(l=60, r=80, t=50, b=80), - width=1100, - height=450, - ) - base.update(kwargs) - return base - - -def _quote_is_usd(pair: str) -> bool: - """Check if a pair's quote currency is a USD stablecoin.""" - if not pair or "-" not in pair: - return False - return pair.split("-")[-1].upper() in ("USD", "USDT", "USDC", "BUSD", "FDUSD") - - -def _vol_to_usd(ctrl: dict, to_usd: float) -> float: - """Convert controller volume to USD based on the pair's quote currency.""" - if _quote_is_usd(ctrl.get("pair", "")): - return ctrl["vol"] - return ctrl["vol"] * to_usd - - -def _amount_quote_to_usd(ctrl: dict, to_usd: float) -> float: - """Convert total_amount_quote to USD based on the pair's quote currency.""" - amt = ctrl.get("total_amount_quote", 0) - if _quote_is_usd(ctrl.get("pair", "")): - return amt - return amt * to_usd - - -def _market_vol_to_usd( - pair: str, - vol_base: float, - vol_quote: float, - to_usd: float, - prices: dict[str, float], -) -> float: - """Convert market candle volume to USD for any pair.""" - base = pair.split("-")[0].upper() if "-" in pair else "" - quote = pair.split("-")[-1].upper() if "-" in pair else "" - - # If base is a stablecoin, vol_base is already ~USD - if base in ("USDT", "USDC", "BUSD", "DAI", "FDUSD"): - return vol_base - - # If quote is USD stablecoin, vol_quote is already USD - if quote in ("USD", "USDT", "USDC", "BUSD", "FDUSD"): - return ( - vol_quote - if vol_quote > 0 - else vol_base * (prices.get(f"{base}-USDT", 0) or 0) - ) - - # Non-USD quote (e.g. BRL) — convert quote volume, or use base price - if vol_quote > 0: - return vol_quote * to_usd - bp = prices.get(f"{base}-USDT", 0) or prices.get(f"{base}-USD", 0) - return vol_base * bp if bp else 0 - - -def _short_chart_name(ctrl_id: str, pair: str = "") -> str: - """Clean controller name for chart labels.""" - name = ctrl_id - # Strip common prefixes - name = re.sub(r"^mm_", "", name) - name = re.sub(r"pmm_?mister_?", "", name) - # Strip connector prefixes (binance_, binance_perpetual_, etc.) - name = re.sub(r"^(binance|okx|bybit|kucoin)(_perpetual)?_", "", name) - # Strip pair fragment - if pair: - pair_frag = pair.replace("-", "_") - name = name.replace(pair_frag, "").replace(pair, "") - while "__" in name: - name = name.replace("__", "_") - name = name.strip("_") - name = re.sub(r"config[_]?(\d+)", r"c\1", name) - return name or ctrl_id[:20] - - -def _build_pnl_chart( - controllers: list[dict], - to_usd: float, - quote_currency: str = "BRL", - ctrl_configs: dict | None = None, -) -> go.Figure: - """Horizontal bar chart: PnL breakdown, Total PnL, and Vol/Capital by controller.""" - # Sort by net PnL - sorted_ctrls = sorted(controllers, key=lambda c: c["net"] * to_usd) - names = [_short_chart_name(c["id"], c["pair"]) for c in sorted_ctrls] - realized = [c["realized"] * to_usd for c in sorted_ctrls] - unrealized = [c["unrealized"] * to_usd for c in sorted_ctrls] - rebates = [ - ( - c["vol"] * to_usd * REBATE_RATE - if _pair_has_rebate(c["pair"], quote_currency) - else 0 - ) - for c in sorted_ctrls - ] - totals = [r + u + rb for r, u, rb in zip(realized, unrealized, rebates)] - volumes = [_vol_to_usd(c, to_usd) for c in sorted_ctrls] - - # Capital from configs (converted to USD) - has_capital = False - capitals = [] - if ctrl_configs: - for c in sorted_ctrls: - cfg = ctrl_configs.get(c["id"], {}) - amt = float(cfg.get("total_amount_quote", 0) or 0) - ctrl_for_conv = {"total_amount_quote": amt, "pair": c["pair"]} - capitals.append(_amount_quote_to_usd(ctrl_for_conv, to_usd)) - if amt > 0: - has_capital = True - - # Vol/Capital turnover ratio - vol_cap_ratios = [] - if has_capital: - for v, cap in zip(volumes, capitals): - vol_cap_ratios.append(v / cap if cap > 0 else 0) - - ncols = 3 if has_capital else 2 - col_widths = [0.40, 0.30, 0.30] if has_capital else [0.55, 0.45] - subtitles = ( - ["PnL Breakdown (USD)", "Total PnL (USD)", "Vol / Capital"] - if has_capital - else ["PnL Breakdown (USD)", "Total PnL (USD)"] - ) - - fig = make_subplots( - rows=1, - cols=ncols, - shared_yaxes=True, - column_widths=col_widths, - horizontal_spacing=0.03, - subplot_titles=subtitles, - ) - - # Col 1: PnL breakdown (stacked) - fig.add_trace( - go.Bar( - y=names, - x=realized, - orientation="h", - name="Realized", - marker_color=_GREEN, - opacity=0.85, - ), - row=1, - col=1, - ) - fig.add_trace( - go.Bar( - y=names, - x=unrealized, - orientation="h", - name="Unrealized", - marker_color=_BLUE, - opacity=0.85, - ), - row=1, - col=1, - ) - fig.add_trace( - go.Bar( - y=names, - x=rebates, - orientation="h", - name="Rebate", - marker_color=_YELLOW, - opacity=0.7, - ), - row=1, - col=1, - ) - - # Col 2: Total PnL (single bar per controller) - total_colors = [_GREEN if t >= 0 else _RED for t in totals] - fig.add_trace( - go.Bar( - y=names, - x=totals, - orientation="h", - name="Total", - marker_color=total_colors, - opacity=0.9, - text=[f"{'+'if t>=0 else ''}{t:.1f}" for t in totals], - textposition="inside", - textfont=dict(size=10, color="white"), - insidetextanchor="middle", - showlegend=False, - ), - row=1, - col=2, - ) - - # Col 3: Vol/Capital turnover - if has_capital: - fig.add_trace( - go.Bar( - y=names, - x=vol_cap_ratios, - orientation="h", - name="Vol/Capital", - marker_color="#8b5cf6", - opacity=0.85, - text=[f"{r:.0f}x" if r > 0 else "-" for r in vol_cap_ratios], - textposition="outside", - textfont=dict(size=10), - showlegend=False, - ), - row=1, - col=3, - ) - - height = max(350, len(names) * 40 + 100) - fig.update_layout( - **_chart_layout("PnL Breakdown · Total · Vol/Capital", height=height), - barmode="relative", - legend=dict(orientation="h", y=-0.12, x=0.5, xanchor="center"), - ) - fig.update_xaxes(gridcolor="#2a2a4a", title="USD", row=1, col=1) - fig.update_xaxes(gridcolor="#2a2a4a", title="USD", row=1, col=2) - if has_capital: - fig.update_xaxes(gridcolor="#2a2a4a", title="Turnover (x)", row=1, col=3) - fig.update_yaxes(gridcolor="#2a2a4a", row=1, col=1) - # Fix subplot title colors - for ann in fig.layout.annotations: - ann.font = dict(color="#aaa", size=11) - return fig - - -def _build_pair_overview_chart( - controllers: list[dict], - to_usd: float, - volumes_since: dict[str, tuple[float, float]], - prices: dict[str, float], - quote_currency: str = "BRL", -) -> go.Figure: - """Unified horizontal bar chart: PnL, Volume, and Market Share by pair.""" - # Aggregate data per pair - pair_data: dict[str, dict] = defaultdict( - lambda: {"realized": 0, "unrealized": 0, "vol": 0} - ) - for c in controllers: - key = c["pair"] or "unknown" - pair_data[key]["realized"] += c["realized"] * to_usd - pair_data[key]["unrealized"] += c["unrealized"] * to_usd - pair_data[key]["vol"] += _vol_to_usd(c, to_usd) - - # Sort pairs by total volume (highest at top) - pairs = sorted(pair_data.keys(), key=lambda p: pair_data[p]["vol"]) - - # Compute market share data - pair_market: dict[str, dict] = {} - for pair in pairs: - vol_data = volumes_since.get(pair, (0, 0)) - vol_base, vol_quote = vol_data if isinstance(vol_data, tuple) else (vol_data, 0) - mkt_usd = ( - _market_vol_to_usd(pair, vol_base, vol_quote, to_usd, prices) - if (vol_base > 0 or vol_quote > 0) - else 0 - ) - our_vol = pair_data[pair]["vol"] - share = (our_vol / mkt_usd * 100) if mkt_usd > 0 else 0 - pair_market[pair] = {"market": mkt_usd, "ours": our_vol, "share": share} - - has_market = any(d["market"] > 0 for d in pair_market.values()) - ncols = 3 if has_market else 2 - col_widths = [0.4, 0.35, 0.25] if has_market else [0.5, 0.5] - subtitles = ( - ["PnL (USD)", "Volume (USD)", "Market Share"] - if has_market - else ["PnL (USD)", "Volume (USD)"] - ) - - fig = make_subplots( - rows=1, - cols=ncols, - shared_yaxes=True, - column_widths=col_widths, - horizontal_spacing=0.04, - subplot_titles=subtitles, - ) - - # Col 1: PnL (realized + unrealized + rebate) - fig.add_trace( - go.Bar( - y=pairs, - x=[pair_data[p]["realized"] for p in pairs], - orientation="h", - name="Realized", - marker_color=_GREEN, - opacity=0.85, - ), - row=1, - col=1, - ) - fig.add_trace( - go.Bar( - y=pairs, - x=[pair_data[p]["unrealized"] for p in pairs], - orientation="h", - name="Unrealized", - marker_color=_BLUE, - opacity=0.85, - ), - row=1, - col=1, - ) - fig.add_trace( - go.Bar( - y=pairs, - x=[ - ( - pair_data[p]["vol"] * REBATE_RATE - if _pair_has_rebate(p, quote_currency) - else 0 - ) - for p in pairs - ], - orientation="h", - name="Rebate", - marker_color=_YELLOW, - opacity=0.7, - ), - row=1, - col=1, - ) - - # Col 2: Volume - fig.add_trace( - go.Bar( - y=pairs, - x=[pair_data[p]["vol"] for p in pairs], - orientation="h", - name="Volume", - marker_color="#8b5cf6", - opacity=0.8, - ), - row=1, - col=2, - ) - - # Col 3: Market Share as percentage bar - if has_market: - shares = [pair_market[p]["share"] for p in pairs] - fig.add_trace( - go.Bar( - y=pairs, - x=shares, - orientation="h", - name="Market Share", - marker_color=[_BLUE if s > 0 else "#30363d" for s in shares], - opacity=0.85, - text=[f"{s:.2f}%" if s > 0 else "N/A" for s in shares], - textposition="outside", - textfont=dict(size=11, color=_BLUE), - hovertemplate="%{y}
Share: %{x:.2f}%
" - "Ours: $%{customdata[0]:,.0f}
" - "Market: $%{customdata[1]:,.0f}", - customdata=[ - [pair_market[p]["ours"], pair_market[p]["market"]] for p in pairs - ], - ), - row=1, - col=ncols, - ) - - height = max(350, len(pairs) * 55 + 120) - fig.update_layout( - **_chart_layout("Pair Overview: PnL · Volume · Market Share", height=height), - barmode="relative", - legend=dict(orientation="h", y=-0.12, x=0.5, xanchor="center"), - ) - for col in range(1, ncols + 1): - fig.update_xaxes(gridcolor="#2a2a4a", row=1, col=col) - if has_market: - fig.update_xaxes(title="%", row=1, col=ncols) - fig.update_yaxes(gridcolor="#2a2a4a", row=1, col=1) - # Fix subplot title colors - for ann in fig.layout.annotations: - ann.font = dict(color="#aaa", size=11) - return fig - - -# --------------------------------------------------------------------------- -# Config analysis helpers -# --------------------------------------------------------------------------- - - -def _cfg_fmt_value(v) -> str: - if v is None: - return "—" - if isinstance(v, bool): - return "✓" if v else "✗" - if isinstance(v, float): - if v < 0.001: - return f"{v:.5f}" - if v < 0.01: - return f"{v:.4f}" - return f"{v:.3f}" if v < 10 else f"{v:.1f}" - if isinstance(v, list): - return ", ".join(_cfg_fmt_value(x) for x in v) - if isinstance(v, dict): - return " | ".join(f"{k}={_cfg_fmt_value(vv)}" for k, vv in v.items()) - return str(v) - - -def _cfg_normalize(v): - if v is None: - return None - if isinstance(v, str): - try: - return float(v) - except ValueError: - return v.strip().lower() - if isinstance(v, list): - return [_cfg_normalize(x) for x in v] - if isinstance(v, dict): - return {k: _cfg_normalize(vv) for k, vv in v.items()} - return v - - -def _build_config_parallel_coords( - all_configs: dict, running_ids: set -) -> go.Figure | None: - NUMERIC_AXES = [ - ("total_amount_quote", "Capital", None), - ( - "buy_spreads", - "Buy Spread L1 %", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "sell_spreads", - "Sell Spread L1 %", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "buy_spreads", - "Buy Spread L2 %", - lambda v: ( - (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None - ), - ), - ( - "sell_spreads", - "Sell Spread L2 %", - lambda v: ( - (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None - ), - ), - ("executor_refresh_time", "Refresh Time", None), - ("cooldown_time", "Cooldown", None), - ("leverage", "Leverage", None), - ("stop_loss", "Stop Loss", None), - ("take_profit", "Take Profit", None), - ("time_limit", "Time Limit", None), - ("top_executor_refresh_time", "Top Refresh", None), - ( - "dca_spreads", - "DCA Spread L1", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "dca_amounts", - "DCA Amt L1", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "trailing_stop", - "Trail Stop", - lambda v: ( - (v.get("activation_price_delta") if isinstance(v, dict) else v) - if v - else None - ), - ), - ( - "executor_activation_bounds", - "Activation Bounds", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ] - - rows = [] - labels = [] - for label, cfg in all_configs.items(): - clean_label = label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", "") - is_running = clean_label in running_ids or label.startswith("\U0001f7e2") - pair = cfg.get("trading_pair", "unknown") - row = {"_running": 1 if is_running else 0, "_pair": pair} - for param, axis_name, extractor in NUMERIC_AXES: - raw = cfg.get(param) - val = extractor(raw) if extractor else raw - try: - row[axis_name] = float(val) if val is not None else None - except (TypeError, ValueError): - row[axis_name] = None - rows.append(row) - labels.append(_short_chart_name(clean_label, pair)) - - if len(rows) < 2: - return None - - valid_axes = [] - for _, axis_name, _ in NUMERIC_AXES: - vals = [r[axis_name] for r in rows if r[axis_name] is not None] - if len(vals) >= 2 and len(set(vals)) > 1: - valid_axes.append(axis_name) - - if len(valid_axes) < 2: - return None - - unique_pairs = sorted(set(r["_pair"] for r in rows)) - pair_to_idx = {p: i for i, p in enumerate(unique_pairs)} - color_values = [pair_to_idx[r["_pair"]] for r in rows] - - n_pairs = len(unique_pairs) - if n_pairs == 1: - colorscale = [[0, _BLUE], [1, _BLUE]] - else: - colorscale = [ - [i / (n_pairs - 1), _COLORS[i % len(_COLORS)]] for i in range(n_pairs) - ] - - for axis_name in valid_axes: - vals = [r[axis_name] for r in rows if r[axis_name] is not None] - fill = min(vals) if vals else 0 - for r in rows: - if r[axis_name] is None: - r[axis_name] = fill - - dimensions = [] - for axis_name in valid_axes: - vals = [r[axis_name] for r in rows] - if "%" in axis_name: - vals = [v * 100 for v in vals] - dimensions.append( - dict( - label=axis_name, - values=vals, - range=( - [min(vals), max(vals)] - if min(vals) != max(vals) - else [min(vals) - 1, max(vals) + 1] - ), - ) - ) - - fig = go.Figure( - data=go.Parcoords( - line=dict(color=color_values, colorscale=colorscale, showscale=False), - dimensions=dimensions, - customdata=labels, - ) - ) - for i, pair in enumerate(unique_pairs): - fig.add_trace( - go.Scatter( - x=[None], - y=[None], - mode="markers", - marker=dict(size=10, color=_COLORS[i % len(_COLORS)]), - name=pair, - showlegend=True, - ) - ) - fig.update_layout( - **_chart_layout( - f"Config Parameter Space ({len(rows)} configs, {len(valid_axes)} axes)", - width=1400, - height=600, - ), - legend=dict( - orientation="h", y=-0.05, x=0.5, xanchor="center", font=dict(size=12) - ), - ) - return fig - - -def _build_config_spread_chart(all_configs: dict) -> go.Figure | None: - def _to_list(v) -> list: - if v is None: - return [] - if isinstance(v, (int, float)): - return [float(v)] - if isinstance(v, list): - return [float(x) for x in v] - return [] - - items = [] - for label, cfg in all_configs.items(): - buy = _to_list(cfg.get("buy_spreads")) - sell = _to_list(cfg.get("sell_spreads")) - if buy or sell: - short = _short_chart_name( - label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", ""), - cfg.get("trading_pair", ""), - ) - items.append( - { - "name": short, - "pair": cfg.get("trading_pair", ""), - "buy_l1": buy[0] * 100 if len(buy) >= 1 else 0, - "sell_l1": sell[0] * 100 if len(sell) >= 1 else 0, - "buy_l2": buy[1] * 100 if len(buy) >= 2 else 0, - "sell_l2": sell[1] * 100 if len(sell) >= 2 else 0, - } - ) - - if not items: - return None - - items.sort(key=lambda c: (c["pair"], c["name"])) - names = [c["name"] for c in items] - - fig = go.Figure() - fig.add_trace( - go.Bar( - x=names, - y=[c["buy_l1"] for c in items], - name="Buy L1", - marker_color=_GREEN, - opacity=0.9, - ) - ) - fig.add_trace( - go.Bar( - x=names, - y=[c["sell_l1"] for c in items], - name="Sell L1", - marker_color=_RED, - opacity=0.9, - ) - ) - if any(c["buy_l2"] > 0 or c["sell_l2"] > 0 for c in items): - fig.add_trace( - go.Bar( - x=names, - y=[c["buy_l2"] for c in items], - name="Buy L2", - marker_color=_GREEN, - opacity=0.5, - ) - ) - fig.add_trace( - go.Bar( - x=names, - y=[c["sell_l2"] for c in items], - name="Sell L2", - marker_color=_RED, - opacity=0.5, - ) - ) - - fig.update_layout( - **_chart_layout("Spread Levels by Controller (%)", width=1200, height=500), - barmode="group", - yaxis=dict(gridcolor="#2a2a4a", title="Spread %"), - xaxis=dict(gridcolor="#2a2a4a", tickangle=-45), - legend=dict(orientation="h", y=1.1, x=0.5, xanchor="center"), - ) - fig.update_layout(margin=dict(l=60, r=40, t=60, b=120)) - return fig - - -def _build_config_capital_chart(all_configs: dict) -> go.Figure | None: - data = [] - for label, cfg in all_configs.items(): - amt = float(cfg.get("total_amount_quote", 0) or 0) - if amt <= 0: - continue - short = _short_chart_name( - label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", ""), - cfg.get("trading_pair", ""), - ) - data.append({"name": short, "pair": cfg.get("trading_pair", ""), "amount": amt}) - - if not data: - return None - - data.sort(key=lambda d: (d["pair"], d["amount"])) - unique_pairs = list(dict.fromkeys(d["pair"] for d in data)) - pair_colors = {p: _COLORS[i % len(_COLORS)] for i, p in enumerate(unique_pairs)} - - fig = go.Figure() - for pair in unique_pairs: - pair_data = [d for d in data if d["pair"] == pair] - fig.add_trace( - go.Bar( - y=[d["name"] for d in pair_data], - x=[d["amount"] for d in pair_data], - orientation="h", - name=pair, - marker_color=pair_colors[pair], - opacity=0.85, - text=[f"{d['amount']:,.0f}" for d in pair_data], - textposition="auto", - ) - ) - - height = max(400, len(data) * 22 + 100) - fig.update_layout( - **_chart_layout( - "Capital Allocation by Controller (quote)", width=1200, height=height - ), - barmode="stack", - xaxis=dict(gridcolor="#2a2a4a", title="Amount (quote)"), - yaxis=dict(gridcolor="#2a2a4a"), - legend=dict(orientation="h", y=1.05, x=0.5, xanchor="center"), - ) - fig.update_layout(margin=dict(l=150, r=40, t=60, b=50)) - return fig - - -# --------------------------------------------------------------------------- -# Report builder -# --------------------------------------------------------------------------- - - -def _build_report( - bots_data: dict, - bot_runs: dict, - prices: dict[str, float], - volumes_since: dict[str, tuple[float, float]], - quote_usd_rate: float, - quote_currency: str = "BRL", - ctrl_configs: dict | None = None, - balances: dict | None = None, -) -> list[str]: - ctrl_configs = ctrl_configs or {} - - # ---- Parse controller data ---- - controllers = [] - - for bot_name, bot_data in bots_data.items(): - if isinstance(bot_data, str): - continue - - performance = {} - if isinstance(bot_data, dict): - performance = bot_data.get("performance", {}) - elif isinstance(bot_data, list): - performance = { - f"ctrl_{i}": {"performance": c} for i, c in enumerate(bot_data) - } - - for ctrl_name, ctrl_data in performance.items(): - if not isinstance(ctrl_data, dict): - continue - perf = ctrl_data.get("performance", ctrl_data) - cfg = ctrl_configs.get(ctrl_name, {}) - trading_pair = cfg.get("trading_pair", "") or perf.get("trading_pair", "") - - vol = float(perf.get("volume_traded", 0) or 0) - realized = float(perf.get("realized_pnl_quote", 0) or 0) - unrealized = float(perf.get("unrealized_pnl_quote", 0) or 0) - total_amount_quote = float(cfg.get("total_amount_quote", 0) or 0) - - controllers.append( - { - "id": ctrl_name, - "bot": bot_name, - "pair": trading_pair, - "vol": vol, - "realized": realized, - "unrealized": unrealized, - "net": realized + unrealized, - "total_amount_quote": total_amount_quote, - } - ) - - if not controllers: - return ["No controller data found\\."] - - # ---- Runtime ---- - earliest_dt = None - for bn in {c["bot"] for c in controllers}: - ts = bot_runs.get(bn) - if ts: - try: - ts_str = str(ts) - if ts_str.endswith("Z"): - ts_str = ts_str[:-1] + "+00:00" - dt = datetime.fromisoformat(ts_str) - if earliest_dt is None or dt < earliest_dt: - earliest_dt = dt - except (ValueError, TypeError): - pass - - now_dt = datetime.now(timezone.utc) - runtime_h = (now_dt - earliest_dt).total_seconds() / 3600 if earliest_dt else 0 - if earliest_dt: - start_str = earliest_dt.strftime("%b %d, %H:%M UTC") - else: - start_str = "N/A" - - # ---- Totals ---- - total_vol_q = sum(c["vol"] for c in controllers) - total_realized_q = sum(c["realized"] for c in controllers) - total_unrealized_q = sum(c["unrealized"] for c in controllers) - total_net_q = total_realized_q + total_unrealized_q - - to_usd = quote_usd_rate - total_vol = sum(_vol_to_usd(c, to_usd) for c in controllers) - total_realized = total_realized_q * to_usd - total_unrealized = total_unrealized_q * to_usd - total_net = total_net_q * to_usd - total_rebate_vol = sum( - _vol_to_usd(c, to_usd) - for c in controllers - if _pair_has_rebate(c["pair"], quote_currency) - ) - total_rebate = total_rebate_vol * REBATE_RATE - - vol_hr = total_vol / runtime_h if runtime_h > 0 else 0 - realized_hr = total_realized / runtime_h if runtime_h > 0 else 0 - rebate_hr = total_rebate / runtime_h if runtime_h > 0 else 0 - net_after_rebate = total_net + total_rebate - - # ---- Market volumes ---- - unique_pairs = {c["pair"] for c in controllers if c["pair"]} - market_lines = [] - for pair in sorted(unique_pairs): - vol_data = volumes_since.get(pair, (0, 0)) - vol_24h_base, vol_24h_quote = ( - vol_data if isinstance(vol_data, tuple) else (vol_data, 0) - ) - if vol_24h_base <= 0 and vol_24h_quote <= 0: - continue - mkt_usd = _market_vol_to_usd(pair, vol_24h_base, vol_24h_quote, to_usd, prices) - our_vol = sum(_vol_to_usd(c, to_usd) for c in controllers if c["pair"] == pair) - share = (our_vol / mkt_usd * 100) if mkt_usd > 0 else 0 - market_lines.append( - f" {_md_escape(pair)}: mkt \\${_md_escape(_fmt(mkt_usd))} " - f"\\| ours \\${_md_escape(_fmt(our_vol))} \\({_md_escape(f'{share:.2f}')}%\\)" - ) - - # ---- FX rate display ---- - qc = quote_currency.upper() - if qc not in ("USD", "USDT"): - fx_val = _fmt(1 / to_usd if to_usd > 0 else 0) - fx_display = f"1 USD \\= {_md_escape(fx_val)} {_md_escape(qc)}" - else: - fx_display = "" - - # ---- MSG 1: Overview ---- - num_bots = len({c["bot"] for c in controllers}) - num_ctrls = len(controllers) - - # ---- Balance summary ---- - balance_lines = [] - total_balance_usd = 0.0 - if balances: - token_totals: dict[str, float] = defaultdict(float) - token_values: dict[str, float] = defaultdict(float) - for account_data in balances.values(): - for connector_balances in account_data.values(): - if not isinstance(connector_balances, list): - continue - for b in connector_balances: - token = b.get("token", "") - value = float(b.get("value", 0) or 0) - units = float(b.get("units", 0) or 0) - if value > 0: - token_totals[token] += units - token_values[token] += value - total_balance_usd = sum(token_values.values()) - # Top tokens by value - sorted_tokens = sorted(token_values.items(), key=lambda x: x[1], reverse=True) - for token, value in sorted_tokens[:5]: - pct = (value / total_balance_usd * 100) if total_balance_usd > 0 else 0 - balance_lines.append( - f" {_md_escape(token)}: \\${_md_escape(_fmt(value))} \\({_md_escape(f'{pct:.0f}')}%\\)" - ) - - msg1_parts = [ - f"*📊 TRADING REPORT*", - f"━━━━━━━━━━━━━━━━━━━━━━━━", - f"", - f"⏱ {_md_escape(f'{runtime_h:.1f}')}h runtime \\(since {_md_escape(start_str)}\\)", - f"🤖 {num_bots} bots \\| {num_ctrls} controllers", - ] - if fx_display: - msg1_parts.append(f"💱 {fx_display}") - - if balance_lines: - msg1_parts += [ - f"", - f"💰 *Balance: \\${_md_escape(_fmt(total_balance_usd))}*", - ] + balance_lines - - msg1_parts += [ - f"", - f"{_emoji(total_net)} *Performance \\(USD\\)*", - f"Volume: \\${_md_escape(_fmt(total_vol))}", - f"Realized: {_md_escape(_sign(total_realized))}\\${_md_escape(_fmt(abs(total_realized)))}", - f"Unrealized: {_md_escape(_sign(total_unrealized))}\\${_md_escape(_fmt(abs(total_unrealized)))}", - f"Net PnL: {_md_escape(_sign(total_net))}\\${_md_escape(_fmt(abs(total_net)))}", - f"Rebate: \\+\\${_md_escape(_fmt(total_rebate))}", - f"Net\\+Rebate: {_md_escape(_sign(net_after_rebate))}\\${_md_escape(_fmt(abs(net_after_rebate)))}", - f"", - f"⏳ *Hourly Rates*", - f"Vol/hr: \\${_md_escape(_fmt(vol_hr))}", - f"Realized/hr: {_md_escape(_sign(realized_hr))}\\${_md_escape(_fmt(abs(realized_hr)))}", - f"Rebate/hr: \\+\\${_md_escape(_fmt(rebate_hr))}", - ] - - if market_lines: - msg1_parts += [f"", f"📈 *Market Share*"] + market_lines - - # ---- MSG 2: Controllers grouped by pair ---- - msg2_parts = [ - f"*📋 CONTROLLERS BY PAIR*", - f"━━━━━━━━━━━━━━━━━━━━━━━━", - ] - - pair_groups: dict[str, list[dict]] = defaultdict(list) - for c in controllers: - pair_groups[c["pair"] or "unknown"].append(c) - - sorted_pairs = sorted( - pair_groups.items(), key=lambda x: sum(c["vol"] for c in x[1]), reverse=True - ) - - for pair, ctrls in sorted_pairs: - grp_vol = sum(_vol_to_usd(c, to_usd) for c in ctrls) - grp_realized = sum(c["realized"] for c in ctrls) * to_usd - grp_unrealized = sum(c["unrealized"] for c in ctrls) * to_usd - grp_net = grp_realized + grp_unrealized - grp_rebate = ( - grp_vol * REBATE_RATE if _pair_has_rebate(pair, quote_currency) else 0 - ) - grp_adj = grp_net + grp_rebate - - pair_escaped = _md_escape(pair) - msg2_parts.append(f"") - msg2_parts.append( - f"▸ *{pair_escaped}* \\({len(ctrls)} ctrls\\) " - f"{_emoji(grp_adj)} {_md_escape(_sign(grp_adj))}\\${_md_escape(_fmt(abs(grp_adj)))}" - ) - - # Build controller table rows - sorted_ctrls = sorted(ctrls, key=lambda c: c["vol"], reverse=True) - - def _short_name(ctrl_id: str, pair: str) -> str: - name = ctrl_id - if name.startswith("mm_"): - name = name[3:] - # Strip all variations of pmm_mister prefix - name = re.sub(r"pmm_?mister_?", "", name) - pair_frag = pair.replace("-", "_") if pair else "" - if pair_frag: - name = name.replace(pair_frag, "").replace(pair, "") - while "__" in name: - name = name.replace("__", "_") - name = name.strip("_") - name = re.sub(r"config[_]?(\d+)", r"c\1", name) - return name or ctrl_id[:20] - - rows = [] - for c in sorted_ctrls: - vol_usd = _vol_to_usd(c, to_usd) - net_usd = c["net"] * to_usd - reb_usd = ( - vol_usd * REBATE_RATE if _pair_has_rebate(pair, quote_currency) else 0 - ) - adj_usd = net_usd + reb_usd - short = _short_name(c["id"], pair) - rows.append((short, vol_usd, net_usd, adj_usd)) - - # Calculate column widths - name_w = max(len(r[0]) for r in rows) - vol_strs = [f"${_fmt(r[1])}" for r in rows] - pnl_strs = [f"{_sign(r[2])}${_fmt(abs(r[2]))}" for r in rows] - adj_strs = [f"{_sign(r[3])}${_fmt(abs(r[3]))}" for r in rows] - - vol_w = max(len(s) for s in vol_strs) - pnl_w = max(len(s) for s in pnl_strs) - adj_w = max(len(s) for s in adj_strs) - - # Header + separator + rows inside a code block - hdr = ( - f"{'Name':<{name_w}} {'Vol':>{vol_w}} {'PnL':>{pnl_w}} {'Adj':>{adj_w}}" - ) - sep = "─" * len(hdr) - - table_lines = [hdr, sep] - for i, (short, vol_usd, net_usd, adj_usd) in enumerate(rows): - e = "+" if adj_usd >= 0 else "-" - table_lines.append( - f"{short:<{name_w}} {vol_strs[i]:>{vol_w}} {pnl_strs[i]:>{pnl_w}} {adj_strs[i]:>{adj_w}}" - ) - - # Pair subtotal row - sub_vol = f"${_fmt(grp_vol)}" - sub_pnl = f"{_sign(grp_net)}${_fmt(abs(grp_net))}" - sub_adj = f"{_sign(grp_adj)}${_fmt(abs(grp_adj))}" - table_lines.append(sep) - table_lines.append( - f"{'TOTAL':<{name_w}} {sub_vol:>{vol_w}} {sub_pnl:>{pnl_w}} {sub_adj:>{adj_w}}" - ) - - msg2_parts.append("```") - msg2_parts.extend(table_lines) - msg2_parts.append("```") - - return ["\n".join(msg1_parts), "\n".join(msg2_parts)] - - -# --------------------------------------------------------------------------- -# Telegram HTTP fallback (when context.bot is unavailable) -# --------------------------------------------------------------------------- - - -async def _send_telegram_http( - chat_id: int, text: str, parse_mode: str = "MarkdownV2" -) -> bool: - """Send a message via Telegram HTTP API directly. Used as fallback.""" - token = os.environ.get("TELEGRAM_BOT_TOKEN", "") - if not token: - logger.error("TELEGRAM_BOT_TOKEN not set, cannot send via HTTP fallback") - return False - try: - import httpx - - url = f"https://api.telegram.org/bot{token}/sendMessage" - payload = {"chat_id": chat_id, "text": text, "parse_mode": parse_mode} - async with httpx.AsyncClient(timeout=10) as http_client: - resp = await http_client.post(url, json=payload) - data = resp.json() - if data.get("ok"): - return True - # Retry without parse_mode if formatting fails - if "can't parse" in data.get("description", "").lower(): - payload.pop("parse_mode") - resp = await http_client.post(url, json=payload) - data = resp.json() - if data.get("ok"): - return True - logger.error(f"Telegram HTTP send failed: {data.get('description')}") - except Exception as e: - logger.error(f"Telegram HTTP fallback error: {e}") - return False - - -# --------------------------------------------------------------------------- -# Main entry point -# --------------------------------------------------------------------------- - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: - chat_id = context._chat_id if hasattr(context, "_chat_id") else None - if not chat_id: - return "No chat_id available" - - # Auto-resolve target_chat_id from notes if not explicitly set - send_to = config.target_chat_id - if not send_to: - try: - from condor.preferences import get_note - - user_data = context.user_data if hasattr(context, "user_data") else {} - # Look up note for the active server's group chat - server_name = ( - config.server_name - or get_config_manager().get_chat_default_server(chat_id) - ) - if server_name: - note_val = get_note(user_data, f"server.{server_name}.group_chat_id") - if note_val: - send_to = int(note_val) - except Exception: - pass - if not send_to: - # Fallback to CONDOR_CHAT_ID env var (the invoking user's chat) - env_chat = os.environ.get("CONDOR_CHAT_ID", "") - send_to = int(env_chat) if env_chat else chat_id - - # Connect to the specified server (default: brigado_2 where bots run) - if config.server_name: - client = await get_config_manager().get_client(config.server_name) - else: - client = await get_client(chat_id, context=context) - if not client: - return "Could not connect to API server" - - bots_data, bot_runs, balances = await asyncio.gather( - _get_bots_data(client), - _get_bot_runs(client), - _get_balances(client), - ) - - if not bots_data: - await context.bot.send_message(chat_id=send_to, text="No active bots found.") - return "No active bots" - - # Fetch controller configs for trading_pair - ctrl_configs = {} - for bot_name in bots_data: - try: - configs = await client.controllers.get_bot_controller_configs(bot_name) - if isinstance(configs, list): - for cfg in configs: - cid = cfg.get("id") or cfg.get("controller_id", "") - if cid: - ctrl_configs[cid] = cfg - except Exception: - pass - - # Determine trading pairs and connector - trading_pairs = set() - connector = "binance" - for bot_data in bots_data.values(): - if not isinstance(bot_data, dict): - continue - for ctrl_name, ctrl_data in bot_data.get("performance", {}).items(): - if not isinstance(ctrl_data, dict): - continue - cfg = ctrl_configs.get(ctrl_name, {}) - perf = ctrl_data.get("performance", ctrl_data) - pair = cfg.get("trading_pair", "") or perf.get("trading_pair", "") - if pair: - trading_pairs.add(pair) - conn = cfg.get("connector_name", "") or perf.get("connector_name", "") - if conn: - connector = conn.split("_")[0] - - # Build price pairs for USD conversion - price_pairs = list(trading_pairs) - for bt in {p.split("-")[0] for p in trading_pairs if "-" in p}: - if f"{bt}-USDT" not in price_pairs: - price_pairs.append(f"{bt}-USDT") - for qt in {p.split("-")[-1] for p in trading_pairs if "-" in p}: - if qt not in ("USDT", "USD") and f"USDT-{qt}" not in price_pairs: - price_pairs.append(f"USDT-{qt}") - - prices = await _get_prices(client, connector, price_pairs) - - # FX rate fallback - qc = config.quote_currency.upper() - fx_pair = f"USDT-{qc}" if qc not in ("USD", "USDT") else "" - if fx_pair and not prices.get(fx_pair): - fx_prices = await _get_prices(client, "binance", [fx_pair]) - prices.update(fx_prices) - - # Market volumes since deploy - earliest_dt = None - for bn in bots_data: - ts = bot_runs.get(bn) - if ts: - try: - ts_str = str(ts) - if ts_str.endswith("Z"): - ts_str = ts_str[:-1] + "+00:00" - dt = datetime.fromisoformat(ts_str) - if earliest_dt is None or dt < earliest_dt: - earliest_dt = dt - except (ValueError, TypeError): - pass - - volumes_since = {} - vol_tasks = { - pair: _get_market_volume_since(client, connector, pair, earliest_dt) - for pair in trading_pairs - } - vol_results = await asyncio.gather(*vol_tasks.values()) - for pair, vol_tuple in zip(vol_tasks.keys(), vol_results): - volumes_since[pair] = vol_tuple - - # Quote → USD rate - quote_usd_rate = 1.0 - if qc not in ("USD", "USDT"): - usdt_quote = prices.get(f"USDT-{qc}", 0) - if usdt_quote and usdt_quote > 0: - quote_usd_rate = 1.0 / usdt_quote - else: - quote_usdt = prices.get(f"{qc}-USDT", 0) - if quote_usdt and quote_usdt > 0: - quote_usd_rate = quote_usdt - - # Build and send - messages = _build_report( - bots_data, - bot_runs, - prices, - volumes_since, - quote_usd_rate, - quote_currency=qc, - ctrl_configs=ctrl_configs, - balances=balances, - ) - - sent = 0 - for i, msg in enumerate(messages): - if i > 0: - await asyncio.sleep(1) - for attempt in range(3): - try: - await context.bot.send_message( - chat_id=send_to, - text=msg, - parse_mode="MarkdownV2", - ) - sent += 1 - break - except Exception as e: - err = str(e).lower() - if "flood control" in err or "retry in" in err: - match = re.search(r"retry in (\d+)", err) - wait = int(match.group(1)) + 1 if match else 10 - await asyncio.sleep(wait) - else: - logger.error(f"Failed to send msg {i} via bot: {e}") - # Fallback: send via HTTP directly to Telegram API - if await _send_telegram_http(send_to, msg): - sent += 1 - break - - # Strip MarkdownV2 escaping for plain-text return - def _strip_md(text: str) -> str: - return text.replace("\\", "") - - plain_report = "\n\n".join(_strip_md(m) for m in messages) - - from condor.reports import ReportBuilder - - builder = ReportBuilder("Trading Report") - builder.source("routine", "bot_report").tags(["bots", "performance"]) - builder.manual_order() - - # ---- Parse controller data for charts ---- - controllers_parsed = [] - for bot_name, bot_data in bots_data.items(): - if isinstance(bot_data, str): - continue - performance = {} - if isinstance(bot_data, dict): - performance = bot_data.get("performance", {}) - elif isinstance(bot_data, list): - performance = { - f"ctrl_{i}": {"performance": c} for i, c in enumerate(bot_data) - } - for ctrl_name, ctrl_data in performance.items(): - if not isinstance(ctrl_data, dict): - continue - perf = ctrl_data.get("performance", ctrl_data) - cfg = ctrl_configs.get(ctrl_name, {}) - trading_pair = cfg.get("trading_pair", "") or perf.get("trading_pair", "") - vol = float(perf.get("volume_traded", 0) or 0) - realized = float(perf.get("realized_pnl_quote", 0) or 0) - unrealized = float(perf.get("unrealized_pnl_quote", 0) or 0) - total_amount_quote = float(cfg.get("total_amount_quote", 0) or 0) - controllers_parsed.append( - { - "id": ctrl_name, - "bot": bot_name, - "pair": trading_pair, - "vol": vol, - "realized": realized, - "unrealized": unrealized, - "net": realized + unrealized, - "total_amount_quote": total_amount_quote, - } - ) - - # ---- Compute totals for KPIs ---- - total_vol_usd = sum(_vol_to_usd(c, quote_usd_rate) for c in controllers_parsed) - total_realized_usd = sum(c["realized"] for c in controllers_parsed) * quote_usd_rate - total_unrealized_usd = ( - sum(c["unrealized"] for c in controllers_parsed) * quote_usd_rate - ) - total_net_usd = total_realized_usd + total_unrealized_usd - rebate_vol_usd = sum( - _vol_to_usd(c, quote_usd_rate) - for c in controllers_parsed - if _pair_has_rebate(c["pair"], qc) - ) - total_rebate_usd = rebate_vol_usd * REBATE_RATE - net_after_rebate_usd = total_net_usd + total_rebate_usd - num_bots = len({c["bot"] for c in controllers_parsed}) - num_ctrls = len(controllers_parsed) - - # Runtime - earliest_deploy = None - for bn in {c["bot"] for c in controllers_parsed}: - ts = bot_runs.get(bn) - if ts: - try: - ts_str = str(ts) - if ts_str.endswith("Z"): - ts_str = ts_str[:-1] + "+00:00" - dt = datetime.fromisoformat(ts_str) - if earliest_deploy is None or dt < earliest_deploy: - earliest_deploy = dt - except (ValueError, TypeError): - pass - now_utc = datetime.now(timezone.utc) - runtime_h = ( - (now_utc - earliest_deploy).total_seconds() / 3600 if earliest_deploy else 0 - ) - - # Balance - total_balance_usd = 0.0 - if balances: - for account_data in balances.values(): - for connector_balances in account_data.values(): - if not isinstance(connector_balances, list): - continue - for b in connector_balances: - total_balance_usd += float(b.get("value", 0) or 0) - - # ---- KPI cards ---- - pnl_trend = "up" if net_after_rebate_usd >= 0 else "down" - builder.kpi( - "Net PnL + Rebate", - f"{'+'if net_after_rebate_usd>=0 else ''}${_fmt(abs(net_after_rebate_usd))}", - delta=f"${_fmt(abs(net_after_rebate_usd / runtime_h) if runtime_h else 0)}/hr", - trend=pnl_trend, - ) - total_capital_usd = sum( - _amount_quote_to_usd(c, quote_usd_rate) for c in controllers_parsed - ) - builder.kpi( - "Volume", - f"${_fmt(total_vol_usd)}", - delta=f"${_fmt(total_vol_usd / runtime_h if runtime_h else 0)}/hr", - ) - builder.kpi( - "Rebate", - f"${_fmt(total_rebate_usd)}", - delta=f"${_fmt(total_rebate_usd / runtime_h if runtime_h else 0)}/hr", - trend="up", - ) - if total_capital_usd > 0: - builder.kpi( - "Capital Deployed", - f"${_fmt(total_capital_usd)}", - delta=f"{num_ctrls} controllers", - ) - builder.kpi("Balance", f"${_fmt(total_balance_usd)}") - builder.kpi( - "Runtime", - f"{runtime_h:.1f}h", - delta=f"{num_bots} bots, {num_ctrls} controllers", - ) - - # ---- Charts ---- - if controllers_parsed: - builder.plotly( - _build_pair_overview_chart( - controllers_parsed, quote_usd_rate, volumes_since, prices, qc - ) - ) - builder.plotly( - _build_pnl_chart(controllers_parsed, quote_usd_rate, qc, ctrl_configs) - ) - - # ---- Controller details table ---- - ctrl_rows = [] - for c in sorted( - controllers_parsed, key=lambda x: x["net"] * quote_usd_rate, reverse=True - ): - vol_usd = _vol_to_usd(c, quote_usd_rate) - net_usd = c["net"] * quote_usd_rate - reb_usd = vol_usd * REBATE_RATE if _pair_has_rebate(c["pair"], qc) else 0 - adj_usd = net_usd + reb_usd - cap_usd = _amount_quote_to_usd(c, quote_usd_rate) - ctrl_rows.append( - { - "Controller": _short_chart_name(c["id"], c["pair"]), - "Pair": c["pair"], - "Capital": f"${_fmt(cap_usd)}" if cap_usd > 0 else "-", - "Volume": f"${_fmt(vol_usd)}", - "Realized": f"{'+'if c['realized']*quote_usd_rate>=0 else '-'}${_fmt(abs(c['realized']*quote_usd_rate))}", - "Unrealized": f"{'+'if c['unrealized']*quote_usd_rate>=0 else '-'}${_fmt(abs(c['unrealized']*quote_usd_rate))}", - "Rebate": f"+${_fmt(reb_usd)}", - "Total": f"{'+'if adj_usd>=0 else '-'}${_fmt(abs(adj_usd))}", - } - ) - if ctrl_rows: - builder.table(ctrl_rows) - - # ---- Pair summary table ---- - pair_groups: dict[str, list[dict]] = defaultdict(list) - for c in controllers_parsed: - pair_groups[c["pair"] or "unknown"].append(c) - pair_rows = [] - for pair, ctrls in sorted( - pair_groups.items(), key=lambda x: sum(c["vol"] for c in x[1]), reverse=True - ): - grp_vol = sum(_vol_to_usd(c, quote_usd_rate) for c in ctrls) - grp_realized = sum(c["realized"] for c in ctrls) * quote_usd_rate - grp_unrealized = sum(c["unrealized"] for c in ctrls) * quote_usd_rate - grp_net = grp_realized + grp_unrealized - grp_rebate = grp_vol * REBATE_RATE if _pair_has_rebate(pair, qc) else 0 - grp_adj = grp_net + grp_rebate - # Market share - vol_data = volumes_since.get(pair, (0, 0)) - vol_base, vol_quote = vol_data if isinstance(vol_data, tuple) else (vol_data, 0) - mkt_usd = _market_vol_to_usd(pair, vol_base, vol_quote, quote_usd_rate, prices) - share = f"{grp_vol / mkt_usd * 100:.2f}%" if mkt_usd > 0 else "N/A" - grp_capital = sum(_amount_quote_to_usd(c, quote_usd_rate) for c in ctrls) - pair_rows.append( - { - "Pair": pair, - "Controllers": str(len(ctrls)), - "Capital": f"${_fmt(grp_capital)}" if grp_capital > 0 else "-", - "Volume": f"${_fmt(grp_vol)}", - "Realized": f"{'+'if grp_realized>=0 else '-'}${_fmt(abs(grp_realized))}", - "Net PnL": f"{'+'if grp_net>=0 else '-'}${_fmt(abs(grp_net))}", - "Rebate": f"+${_fmt(grp_rebate)}", - "Adjusted": f"{'+'if grp_adj>=0 else '-'}${_fmt(abs(grp_adj))}", - "Mkt Share": share, - } - ) - if pair_rows: - builder.table(pair_rows) - - # ---- Balance table ---- - if balances: - token_totals: dict[str, float] = defaultdict(float) - token_values: dict[str, float] = defaultdict(float) - for account_data in balances.values(): - for connector_balances in account_data.values(): - if not isinstance(connector_balances, list): - continue - for b in connector_balances: - token = b.get("token", "") - value = float(b.get("value", 0) or 0) - units = float(b.get("units", 0) or 0) - if value > 0: - token_totals[token] += units - token_values[token] += value - if token_values: - bal_rows = [] - for token, value in sorted( - token_values.items(), key=lambda x: x[1], reverse=True - ): - pct = (value / total_balance_usd * 100) if total_balance_usd > 0 else 0 - bal_rows.append( - { - "Token": token, - "Amount": _fmt(token_totals[token], 4), - "Value (USD)": f"${_fmt(value)}", - "Weight": f"{pct:.1f}%", - } - ) - builder.table(bal_rows) - - # ---- Config Analysis section ---- - if len(ctrl_configs) > 1: - all_params = set() - for cfg in ctrl_configs.values(): - all_params.update(cfg.keys()) - all_params -= _CONFIG_SKIP_PARAMS - sorted_params = sorted( - all_params, key=lambda p: (0 if p in _CONFIG_KEY_PARAMS else 1, p) - ) - - # Group configs by pair - cfg_pair_groups: dict[str, dict] = defaultdict(dict) - for cid, cfg in ctrl_configs.items(): - pair = cfg.get("trading_pair", "unknown") - cfg_pair_groups[pair][cid] = cfg - - # Count total diffs - total_diffs = 0 - pair_sections = [] - for pair, pair_cfgs in sorted( - cfg_pair_groups.items(), key=lambda x: -len(x[1]) - ): - if len(pair_cfgs) < 2: - continue - labels = list(pair_cfgs.keys()) - shorts = [_short_chart_name(l, pair) for l in labels] - - diff_rows = [] - for param in sorted_params: - values = [_cfg_normalize(pair_cfgs[l].get(param)) for l in labels] - non_none = [v for v in values if v is not None] - if len(non_none) < 2 or len(set(str(v) for v in non_none)) <= 1: - continue - is_key = param in _CONFIG_KEY_PARAMS - row = {"Parameter": f"{'🔴 ' if is_key else '🟡 '}{param}"} - for short, label in zip(shorts, labels): - row[short] = _cfg_fmt_value(pair_cfgs[label].get(param)) - diff_rows.append(row) - - if diff_rows: - total_diffs += len(diff_rows) - pair_sections.append((pair, len(pair_cfgs), diff_rows)) - - if pair_sections: - builder.markdown( - "## Config Analysis\n\nParameter differences across running controllers. 🔴 = key tuning param, 🟡 = other." - ) - for pair, n_cfgs, rows in pair_sections: - builder.markdown( - f"### {pair} — {len(rows)} diffs across {n_cfgs} controllers" - ) - builder.table(rows) - - # Config charts - labeled_configs = {f"\U0001f7e2 {k}": v for k, v in ctrl_configs.items()} - running_ids = set(ctrl_configs.keys()) - - fig_pc = _build_config_parallel_coords(labeled_configs, running_ids) - if fig_pc: - if not pair_sections: - builder.markdown("## Config Analysis") - builder.plotly(fig_pc) - - fig_spread = _build_config_spread_chart(labeled_configs) - if fig_spread: - builder.plotly(fig_spread) - - await builder.save() - - return f"Report sent: {sent}/{len(messages)} messages\n\n{plain_report}" diff --git a/agents/brigado/routines/brl_dashboard.py b/agents/brigado/routines/brl_dashboard.py deleted file mode 100644 index 691844004..000000000 --- a/agents/brigado/routines/brl_dashboard.py +++ /dev/null @@ -1,573 +0,0 @@ -"""Multi-timeframe USDT-BRL technical dashboard with Plotly chart.""" - -CATEGORY = "Market Data" - -import asyncio -import io -import logging -from datetime import datetime, timezone - -import numpy as np -import pandas as pd -import plotly.graph_objects as go -from plotly.subplots import make_subplots -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from config_manager import get_client -from routines.base import RoutineResult - -logger = logging.getLogger(__name__) - - -class Config(BaseModel): - """USDT-BRL multi-timeframe dashboard: intraday, weekly, monthly, quarterly.""" - - trading_pair: str = Field(default="USDT-BRL", description="Trading pair") - connector_name: str = Field(default="binance", description="Exchange connector") - - -# --------------------------------------------------------------------------- -# Indicators -# --------------------------------------------------------------------------- - - -def compute_sma(series: pd.Series, period: int) -> pd.Series: - return series.rolling(window=period, min_periods=period).mean() - - -def compute_ema(series: pd.Series, period: int) -> pd.Series: - return series.ewm(span=period, adjust=False).mean() - - -def compute_rsi(series: pd.Series, period: int = 14) -> pd.Series: - delta = series.diff() - gain = delta.where(delta > 0, 0.0) - loss = (-delta).where(delta < 0, 0.0) - avg_gain = gain.ewm(alpha=1 / period, min_periods=period).mean() - avg_loss = loss.ewm(alpha=1 / period, min_periods=period).mean() - rs = avg_gain / avg_loss.replace(0, np.nan) - rsi = 100 - (100 / (1 + rs)) - return rsi.fillna(50) - - -def find_sr_levels( - highs: pd.Series, lows: pd.Series, window: int = 20 -) -> tuple[list[float], list[float]]: - """Find support/resistance using rolling min/max pivots.""" - n = len(highs) - if n < window * 2: - return [], [] - - h = highs.values - l = lows.values - close = float(highs.iloc[-1] + lows.iloc[-1]) / 2 - - levels: dict[float, int] = {} - for w in (window, window * 2): - if w >= n // 2: - continue - for i in range(w, n - w): - if l[i] == min(l[i - w : i + w + 1]): - lv = round(float(l[i]), 4) - levels[lv] = levels.get(lv, 0) + 1 - if h[i] == max(h[i - w : i + w + 1]): - lv = round(float(h[i]), 4) - levels[lv] = levels.get(lv, 0) + 1 - - if not levels: - return [], [] - - # Cluster nearby levels (within 0.3%) - sorted_levels = sorted(levels.items()) - clusters: list[list[tuple[float, int]]] = [[sorted_levels[0]]] - for price, hits in sorted_levels[1:]: - cluster_avg = np.mean([p for p, _ in clusters[-1]]) - if abs(price - cluster_avg) / cluster_avg < 0.003: - clusters[-1].append((price, hits)) - else: - clusters.append([(price, hits)]) - - merged = [] - for cluster in clusters: - total_hits = sum(h for _, h in cluster) - avg_price = np.average([p for p, _ in cluster], weights=[h for _, h in cluster]) - merged.append((round(float(avg_price), 4), total_hits)) - - supports = sorted([lv for lv, _ in merged if lv < close], reverse=True)[:3] - resistances = sorted([lv for lv, _ in merged if lv >= close])[:3] - return supports, resistances - - -# --------------------------------------------------------------------------- -# Data fetching & processing -# --------------------------------------------------------------------------- - -TIMEFRAMES = [ - {"label": "Intraday (1m)", "interval": "1m", "days": 1, "short_label": "1D"}, - {"label": "Semanal (15m)", "interval": "15m", "days": 7, "short_label": "1W"}, - {"label": "Mensual (1h)", "interval": "1h", "days": 30, "short_label": "1M"}, - {"label": "Trimestral (1d)", "interval": "1d", "days": 90, "short_label": "3M"}, -] - - -async def fetch_candles( - client, connector: str, pair: str, interval: str, days: int -) -> pd.DataFrame: - """Fetch candles and return as DataFrame.""" - result = await client.market_data.get_candles_last_days( - connector, pair, days=days, interval=interval - ) - # API returns a list directly, or a dict with "candles"/"data" key - if isinstance(result, list): - records = result - elif isinstance(result, dict): - records = result.get("data", result.get("candles", [])) - else: - records = [] - if not records: - return pd.DataFrame() - - df = pd.DataFrame(records) - for col in ("open", "high", "low", "close", "volume"): - df[col] = pd.to_numeric(df[col], errors="coerce") - - ts_col = df["timestamp"] - unit = "s" if ts_col.iloc[0] > 1e12 / 1000 else "ms" - df["datetime"] = pd.to_datetime(ts_col, unit=unit, utc=True) - df = df.sort_values("datetime").reset_index(drop=True) - return df - - -def compute_indicators(df: pd.DataFrame) -> dict: - """Compute all indicators for a timeframe.""" - close = df["close"] - sma20 = compute_sma(close, 20) - sma50 = compute_sma(close, 50) - ema9 = compute_ema(close, 9) - ema21 = compute_ema(close, 21) - rsi = compute_rsi(close, 14) - - current = float(close.iloc[-1]) - open_price = float(close.iloc[0]) - change_pct = (current - open_price) / open_price * 100 - - high = float(df["high"].max()) - low = float(df["low"].min()) - - supports, resistances = find_sr_levels(df["high"], df["low"]) - - # Trend direction - sma20_now = sma20.iloc[-1] - sma50_now = sma50.iloc[-1] - if pd.notna(sma20_now) and pd.notna(sma50_now): - if current > sma20_now > sma50_now: - trend = "ALCISTA" - elif current < sma20_now < sma50_now: - trend = "BAJISTA" - else: - trend = "LATERAL" - else: - trend = "N/A" - - return { - "sma20": sma20, - "sma50": sma50, - "ema9": ema9, - "ema21": ema21, - "rsi": rsi, - "current": current, - "change_pct": change_pct, - "high": high, - "low": low, - "supports": supports, - "resistances": resistances, - "trend": trend, - "rsi_now": float(rsi.iloc[-1]) if pd.notna(rsi.iloc[-1]) else 50, - "sma20_now": float(sma20_now) if pd.notna(sma20_now) else current, - "sma50_now": float(sma50_now) if pd.notna(sma50_now) else current, - } - - -# --------------------------------------------------------------------------- -# Chart -# --------------------------------------------------------------------------- - -COLORS = { - "bg_paper": "#0d1117", - "bg_plot": "#161b22", - "grid": "#21262d", - "text": "#c9d1d9", - "text_dim": "#8b949e", - "green": "#3fb950", - "red": "#f85149", - "sma20": "#f0883e", - "sma50": "#58a6ff", - "rsi_line": "#bc8cff", - "support": "#3fb950", - "resistance": "#f85149", -} - - -def build_chart(timeframe_data: list[dict], pair: str) -> io.BytesIO: - """Build a 4x2 dashboard: candlestick+SMA (left col), RSI (right col) per timeframe.""" - fig = make_subplots( - rows=4, - cols=2, - column_widths=[0.78, 0.22], - row_heights=[0.25, 0.25, 0.25, 0.25], - vertical_spacing=0.045, - horizontal_spacing=0.03, - subplot_titles=( - [ - item["label"] - for item in timeframe_data - for _ in range(2) # one title per subplot pair - ] - if False - else None - ), # We'll add custom titles - ) - - current_price = timeframe_data[0]["indicators"]["current"] if timeframe_data else 0 - - for i, item in enumerate(timeframe_data): - row = i + 1 - df = item["df"] - ind = item["indicators"] - tf = item["tf"] - dates = df["datetime"] - - # --- Candlestick --- - fig.add_trace( - go.Candlestick( - x=dates, - open=df["open"], - high=df["high"], - low=df["low"], - close=df["close"], - increasing_line_color=COLORS["green"], - decreasing_line_color=COLORS["red"], - increasing_fillcolor=COLORS["green"], - decreasing_fillcolor=COLORS["red"], - showlegend=False, - name=tf["short_label"], - ), - row=row, - col=1, - ) - - # SMA20 - sma20_valid = ind["sma20"].dropna() - if len(sma20_valid) > 0: - fig.add_trace( - go.Scatter( - x=dates.iloc[sma20_valid.index], - y=sma20_valid, - line=dict(color=COLORS["sma20"], width=1.2), - showlegend=(i == 0), - name="SMA 20", - ), - row=row, - col=1, - ) - - # SMA50 - sma50_valid = ind["sma50"].dropna() - if len(sma50_valid) > 0: - fig.add_trace( - go.Scatter( - x=dates.iloc[sma50_valid.index], - y=sma50_valid, - line=dict(color=COLORS["sma50"], width=1.2), - showlegend=(i == 0), - name="SMA 50", - ), - row=row, - col=1, - ) - - # Support lines - t0, t1 = dates.iloc[0], dates.iloc[-1] - for s in ind["supports"]: - fig.add_shape( - type="line", - x0=t0, - x1=t1, - y0=s, - y1=s, - line=dict(color=COLORS["support"], width=0.7, dash="dot"), - opacity=0.5, - row=row, - col=1, - ) - - # Resistance lines - for r in ind["resistances"]: - fig.add_shape( - type="line", - x0=t0, - x1=t1, - y0=r, - y1=r, - line=dict(color=COLORS["resistance"], width=0.7, dash="dot"), - opacity=0.5, - row=row, - col=1, - ) - - # --- RSI panel --- - rsi_valid = ind["rsi"].dropna() - if len(rsi_valid) > 0: - fig.add_trace( - go.Scatter( - x=dates.iloc[rsi_valid.index], - y=rsi_valid, - line=dict(color=COLORS["rsi_line"], width=1.5), - fill="tozeroy", - fillcolor="rgba(188,140,255,0.08)", - showlegend=False, - ), - row=row, - col=2, - ) - - # Overbought/oversold bands - fig.add_hline( - y=70, - line_dash="dot", - line_color=COLORS["red"], - opacity=0.4, - row=row, - col=2, - ) - fig.add_hline( - y=30, - line_dash="dot", - line_color=COLORS["green"], - opacity=0.4, - row=row, - col=2, - ) - - # --- Row label annotation --- - trend_color = COLORS["green"] if ind["change_pct"] >= 0 else COLORS["red"] - arrow = "+" if ind["change_pct"] >= 0 else "" - label_text = ( - f"{tf['short_label']} {tf['label'].split('(')[1].rstrip(')')}" - f" | {ind['current']:.3f}" - f" | {arrow}{ind['change_pct']:.2f}%" - f" | RSI: {ind['rsi_now']:.0f}" - f" | {ind['trend']}" - ) - fig.add_annotation( - text=label_text, - xref="paper", - yref="paper", - x=0, - y=1 - (i * 0.25) + 0.005, - showarrow=False, - font=dict(size=11, color=COLORS["text"]), - xanchor="left", - yanchor="bottom", - ) - - # --- Header --- - # Get quarterly change - q_change = ( - timeframe_data[3]["indicators"]["change_pct"] if len(timeframe_data) > 3 else 0 - ) - m_change = ( - timeframe_data[2]["indicators"]["change_pct"] if len(timeframe_data) > 2 else 0 - ) - w_change = ( - timeframe_data[1]["indicators"]["change_pct"] if len(timeframe_data) > 1 else 0 - ) - d_change = timeframe_data[0]["indicators"]["change_pct"] if timeframe_data else 0 - - title_text = ( - f"{pair} | {current_price:.3f}" - f" | D: {d_change:+.2f}% W: {w_change:+.2f}% M: {m_change:+.2f}% 3M: {q_change:+.2f}%" - ) - - # --- Layout --- - fig.update_layout( - title=dict(text=title_text, font=dict(color=COLORS["text"], size=14), x=0.5), - paper_bgcolor=COLORS["bg_paper"], - plot_bgcolor=COLORS["bg_plot"], - font=dict(color=COLORS["text_dim"], size=9), - legend=dict( - orientation="h", - yanchor="bottom", - y=1.05, - xanchor="center", - x=0.4, - font=dict(size=10, color=COLORS["text"]), - bgcolor="rgba(0,0,0,0)", - ), - margin=dict(l=55, r=30, t=100, b=30), - width=1300, - height=1400, - xaxis_rangeslider_visible=False, - xaxis3_rangeslider_visible=False, - xaxis5_rangeslider_visible=False, - xaxis7_rangeslider_visible=False, - ) - - # Style axes - for row in range(1, 5): - # Price axis - fig.update_yaxes( - gridcolor=COLORS["grid"], - zerolinecolor=COLORS["grid"], - tickformat=".3f", - row=row, - col=1, - ) - fig.update_xaxes( - gridcolor=COLORS["grid"], showticklabels=(row == 4), row=row, col=1 - ) - # RSI axis - fig.update_yaxes( - gridcolor=COLORS["grid"], - range=[0, 100], - tickvals=[30, 50, 70], - row=row, - col=2, - ) - fig.update_xaxes(gridcolor=COLORS["grid"], showticklabels=False, row=row, col=2) - - buf = io.BytesIO() - fig.write_image(buf, format="png", scale=2) - buf.seek(0) - return buf, fig - - -# --------------------------------------------------------------------------- -# Text summary -# --------------------------------------------------------------------------- - - -def format_summary(timeframe_data: list[dict], pair: str) -> str: - lines = [f"Dashboard {pair}"] - for item in timeframe_data: - ind = item["indicators"] - tf = item["tf"] - arrow = "+" if ind["change_pct"] >= 0 else "" - sup_str = ", ".join(f"{s:.3f}" for s in ind["supports"][:2]) or "N/A" - res_str = ", ".join(f"{r:.3f}" for r in ind["resistances"][:2]) or "N/A" - lines.append( - f"\n{tf['label']}:" - f"\n Precio: {ind['current']:.3f} ({arrow}{ind['change_pct']:.2f}%)" - f"\n Tendencia: {ind['trend']} | RSI: {ind['rsi_now']:.0f}" - f"\n SMA20: {ind['sma20_now']:.3f} | SMA50: {ind['sma50_now']:.3f}" - f"\n Soporte: {sup_str} | Resistencia: {res_str}" - f"\n Rango: {ind['low']:.3f} - {ind['high']:.3f}" - ) - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# Telegram photo fallback -# --------------------------------------------------------------------------- - - -async def _send_photo_http(chat_id: int, buf: io.BytesIO, caption: str) -> None: - import os - - import httpx - - token = os.environ.get("TELEGRAM_BOT_TOKEN") or os.environ.get("TELEGRAM_TOKEN", "") - if not token: - return - - url = f"https://api.telegram.org/bot{token}/sendPhoto" - buf.seek(0) - async with httpx.AsyncClient(timeout=30) as http_client: - resp = await http_client.post( - url, - data={"chat_id": chat_id, "caption": caption}, - files={"photo": ("dashboard.png", buf, "image/png")}, - ) - if not resp.json().get("ok"): - logger.warning(f"sendPhoto failed: {resp.text}") - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: - chat_id = context._chat_id if hasattr(context, "_chat_id") else None - client = await get_client(chat_id, context=context) - if not client: - return "No server available" - - # Fetch all timeframes in parallel - fetch_tasks = [ - fetch_candles( - client, - config.connector_name, - config.trading_pair, - tf["interval"], - tf["days"], - ) - for tf in TIMEFRAMES - ] - results = await asyncio.gather(*fetch_tasks, return_exceptions=True) - - timeframe_data = [] - for tf, result in zip(TIMEFRAMES, results): - if isinstance(result, Exception): - logger.warning(f"Failed to fetch {tf['label']}: {result}") - continue - if result.empty or len(result) < 10: - logger.warning(f"Not enough data for {tf['label']}: {len(result)} candles") - continue - - indicators = compute_indicators(result) - timeframe_data.append( - { - "tf": tf, - "label": tf["label"], - "df": result, - "indicators": indicators, - } - ) - - if not timeframe_data: - return "No candle data available for any timeframe" - - # Generate chart - chart_bytes: bytes | None = None - plotly_fig = None - try: - buf, plotly_fig = build_chart(timeframe_data, config.trading_pair) - chart_bytes = buf.getvalue() - caption = f"{config.trading_pair} Multi-Timeframe Dashboard" - if chat_id: - buf.seek(0) - if context.bot: - await context.bot.send_photo( - chat_id=chat_id, photo=buf, caption=caption - ) - else: - await _send_photo_http(chat_id, buf, caption) - except Exception as e: - logger.warning(f"Chart generation/send failed: {e}") - - text = format_summary(timeframe_data, config.trading_pair) - - from condor.reports import ReportBuilder - - builder = ReportBuilder(f"Dashboard: {config.trading_pair}") - builder.source("routine", "brl_dashboard").tags(["dashboard", config.trading_pair]) - builder.markdown(text) - if plotly_fig is not None: - builder.plotly(plotly_fig) - await builder.save() - - if context.bot and chart_bytes: - return RoutineResult(text=text, chart_image=chart_bytes) - return text diff --git a/agents/brigado/routines/btc_brl_arbitrage.py b/agents/brigado/routines/btc_brl_arbitrage.py deleted file mode 100644 index a51e3ccd1..000000000 --- a/agents/brigado/routines/btc_brl_arbitrage.py +++ /dev/null @@ -1,608 +0,0 @@ -"""BTC-BRL arbitrage: spread analysis + quoting strategy simulation.""" - -CATEGORY = "Market Data" - -import asyncio -import io -import logging - -import numpy as np -import pandas as pd -import plotly.graph_objects as go -from plotly.subplots import make_subplots -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from config_manager import get_client -from routines.base import RoutineResult - -logger = logging.getLogger(__name__) - -COLORS = { - "bg_paper": "#0d1117", - "bg_plot": "#161b22", - "grid": "#21262d", - "text": "#c9d1d9", - "text_dim": "#8b949e", - "usdt": "#f0b90b", - "brl": "#3fb950", - "spread": "#bc8cff", - "long": "#3fb950", - "short": "#f85149", - "pnl": "#58a6ff", - "fees": "#f78166", -} - -LAYOUT_BASE = dict( - paper_bgcolor=COLORS["bg_paper"], - plot_bgcolor=COLORS["bg_plot"], - font=dict(color=COLORS["text_dim"], size=10), - legend=dict( - orientation="h", - yanchor="bottom", - y=1.05, - xanchor="center", - x=0.5, - font=dict(size=10, color=COLORS["text"]), - bgcolor="rgba(0,0,0,0)", - ), - margin=dict(l=60, r=30, t=100, b=40), -) - - -class Config(BaseModel): - """BTC-BRL arbitrage: spread analysis + quoting strategy simulation.""" - - connector_name: str = Field(default="binance", description="Exchange connector") - lookback_seconds: int = Field( - default=3600, description="1s candles to fetch (3600=1h)" - ) - trade_size_usdt: float = Field( - default=100.0, description="Notional per trade in USDT" - ) - fee_bps: float = Field( - default=10.0, description="Round-trip fee in bps (maker+taker)" - ) - min_spread_bps: float = Field(default=2.0, description="Min spread to enter (bps)") - max_position_usdt: float = Field( - default=5000.0, description="Max accumulated position (USDT)" - ) - - -async def _fetch_candles( - client, connector: str, pair: str, interval: str, max_records: int -) -> pd.DataFrame: - result = await client.market_data.get_candles( - connector, pair, interval=interval, max_records=max_records - ) - if isinstance(result, list): - records = result - elif isinstance(result, dict): - records = result.get("data", result.get("candles", [])) - else: - records = [] - if not records: - return pd.DataFrame() - df = pd.DataFrame(records) - for col in ("open", "high", "low", "close", "volume", "quote_asset_volume"): - if col in df.columns: - df[col] = pd.to_numeric(df[col], errors="coerce") - ts = df["timestamp"] - unit = "s" if ts.iloc[0] > 1e12 / 1000 else "ms" - df["datetime"] = pd.to_datetime(ts, unit=unit, utc=True) - return df.sort_values("datetime").reset_index(drop=True) - - -def _style_axes(fig, rows): - for row in range(1, rows + 1): - fig.update_yaxes( - gridcolor=COLORS["grid"], zerolinecolor=COLORS["grid"], row=row, col=1 - ) - fig.update_xaxes(gridcolor=COLORS["grid"], row=row, col=1) - - -def _merge_spread( - price_usdt: pd.DataFrame, price_brl: pd.DataFrame, usdtbrl: pd.DataFrame -) -> pd.DataFrame: - """Merge 1s candles and compute spread %.""" - for df in (price_usdt, price_brl, usdtbrl): - df["ts"] = df["datetime"].dt.floor("s") - - merged = pd.merge( - price_usdt[["ts", "close"]].rename(columns={"close": "btc_usdt"}), - price_brl[["ts", "close"]].rename(columns={"close": "btc_brl"}), - on="ts", - how="inner", - ) - merged = pd.merge( - merged, - usdtbrl[["ts", "close"]].rename(columns={"close": "usdt_brl_rate"}), - on="ts", - how="inner", - ) - merged["btc_brl_usd"] = merged["btc_brl"] / merged["usdt_brl_rate"] - merged["spread_pct"] = ( - (merged["btc_brl_usd"] - merged["btc_usdt"]) / merged["btc_usdt"] - ) * 100 - merged["spread_bps"] = merged["spread_pct"] * 100 - return merged.sort_values("ts").reset_index(drop=True) - - -def simulate_quoting( - merged: pd.DataFrame, - trade_size_usdt: float, - fee_bps: float, - min_spread_bps: float, - max_position_usdt: float, -) -> pd.DataFrame: - """Simulate a one-sided quoting strategy that captures spread. - - Logic: - - When BRL premium (spread > min_spread): sell on BRL side (capture premium) - → accumulate short BRL / long USDT exposure - - When USDT premium (spread < -min_spread): sell on USDT side (capture premium) - → accumulate short USDT / long BRL exposure - - Position flips naturally as spread oscillates - - Fee deducted on each trade - """ - n = len(merged) - fee_pct = fee_bps / 10000 - - # Track state - position_usdt = np.zeros( - n - ) # net position in USDT terms (+ = long BRL, - = short BRL) - cum_captured = np.zeros(n) # cumulative spread captured (before fees) - cum_fees = np.zeros(n) # cumulative fees paid - cum_pnl = np.zeros(n) # net PnL (captured - fees) - trade_count = np.zeros(n, dtype=int) - actions = [""] * n - - pos = 0.0 - captured = 0.0 - fees = 0.0 - trades = 0 - - for i in range(n): - spread_bps = merged["spread_bps"].iloc[i] - abs_spread = abs(spread_bps) - - if abs_spread > min_spread_bps: - # Check position limits - if spread_bps > 0 and pos > -max_position_usdt: - # BRL premium → sell on BRL (go short BRL exposure) - trade_pnl = (abs_spread / 10000) * trade_size_usdt - trade_fee = fee_pct * trade_size_usdt - captured += trade_pnl - fees += trade_fee - pos -= trade_size_usdt - trades += 1 - actions[i] = "sell_brl" - elif spread_bps < 0 and pos < max_position_usdt: - # USDT premium → sell on USDT (go long BRL exposure) - trade_pnl = (abs_spread / 10000) * trade_size_usdt - trade_fee = fee_pct * trade_size_usdt - captured += trade_pnl - fees += trade_fee - pos += trade_size_usdt - trades += 1 - actions[i] = "sell_usdt" - - position_usdt[i] = pos - cum_captured[i] = captured - cum_fees[i] = fees - cum_pnl[i] = captured - fees - trade_count[i] = trades - - merged = merged.copy() - merged["position_usdt"] = position_usdt - merged["cum_captured"] = cum_captured - merged["cum_fees"] = cum_fees - merged["cum_pnl"] = cum_pnl - merged["trade_count"] = trade_count - merged["action"] = actions - - # Mark-to-market: unrealized PnL from position * current spread change - # Position was entered at various spreads; approximate with running avg - merged["unrealized"] = 0.0 - avg_entry_spread = 0.0 - total_pos = 0.0 - for i in range(n): - if actions[i]: - entry_spread = merged["spread_bps"].iloc[i] - trade_pos = ( - trade_size_usdt if actions[i] == "sell_usdt" else -trade_size_usdt - ) - if abs(total_pos + trade_pos) > 0.01: - avg_entry_spread = ( - avg_entry_spread * total_pos + entry_spread * trade_pos - ) / (total_pos + trade_pos) - total_pos += trade_pos - current_spread = merged["spread_bps"].iloc[i] - # If short BRL (pos < 0), we profit when spread decreases - # If long BRL (pos > 0), we profit when spread increases - spread_change = current_spread - avg_entry_spread - merged.loc[merged.index[i], "unrealized"] = (spread_change / 10000) * total_pos - - merged["total_pnl"] = merged["cum_pnl"] + merged["unrealized"] - - return merged - - -def build_spread_chart(merged: pd.DataFrame) -> go.Figure: - """Chart 1: Raw spread analysis.""" - avg_spread = merged["spread_pct"].mean() - curr_spread = merged["spread_pct"].iloc[-1] - std_spread = merged["spread_pct"].std() - max_spread = merged["spread_pct"].max() - min_spread = merged["spread_pct"].min() - current_btc = float(merged["btc_usdt"].iloc[-1]) - - fig = make_subplots( - rows=2, - cols=1, - row_heights=[0.55, 0.45], - vertical_spacing=0.08, - subplot_titles=[ - "BTC: USDT vs BRL→USD (1s)", - f"Spread % — avg {avg_spread:+.4f}%, std {std_spread:.4f}%", - ], - ) - - fig.add_trace( - go.Scatter( - x=merged["ts"], - y=merged["btc_usdt"], - line=dict(color=COLORS["usdt"], width=1.5), - name="BTC-USDT", - ), - row=1, - col=1, - ) - fig.add_trace( - go.Scatter( - x=merged["ts"], - y=merged["btc_brl_usd"], - line=dict(color=COLORS["brl"], width=1.5), - name="BTC-BRL (USD)", - ), - row=1, - col=1, - ) - - # Spread with color fill - fig.add_trace( - go.Scatter( - x=merged["ts"], - y=merged["spread_pct"], - line=dict(color=COLORS["spread"], width=1.2), - fill="tozeroy", - fillcolor="rgba(188,140,255,0.08)", - name="Spread %", - ), - row=2, - col=1, - ) - fig.add_hline(y=0, line_color=COLORS["grid"], row=2, col=1) - fig.add_hline( - y=avg_spread, - line_dash="dash", - line_color=COLORS["text_dim"], - opacity=0.6, - row=2, - col=1, - annotation_text=f"avg {avg_spread:+.4f}%", - annotation_font_color=COLORS["text_dim"], - ) - - fig.update_layout( - **LAYOUT_BASE, - title=dict( - text=( - f"BTC-BRL Spread | BTC: ${current_btc:,.0f}" - f" | Now: {curr_spread:+.4f}%" - f" | Range: [{min_spread:+.4f}%, {max_spread:+.4f}%]" - ), - font=dict(color=COLORS["text"], size=13), - x=0.5, - ), - width=1300, - height=700, - ) - fig.update_yaxes(title_text="USD", tickformat=",.0f", row=1, col=1) - fig.update_yaxes(title_text="%", tickformat=".4f", row=2, col=1) - for ann in fig.layout.annotations: - ann.font = dict(color=COLORS["text_dim"], size=11) - _style_axes(fig, 2) - return fig - - -def build_strategy_chart( - sim: pd.DataFrame, fee_bps: float, min_spread_bps: float, trade_size_usdt: float -) -> go.Figure: - """Chart 2: Strategy simulation results.""" - total_trades = int(sim["trade_count"].iloc[-1]) - final_pnl = sim["cum_pnl"].iloc[-1] - final_captured = sim["cum_captured"].iloc[-1] - final_fees = sim["cum_fees"].iloc[-1] - final_position = sim["position_usdt"].iloc[-1] - final_total = sim["total_pnl"].iloc[-1] - duration_min = (sim["ts"].iloc[-1] - sim["ts"].iloc[0]).total_seconds() / 60 - - fig = make_subplots( - rows=3, - cols=1, - row_heights=[0.4, 0.3, 0.3], - vertical_spacing=0.06, - subplot_titles=[ - f"Cumulative PnL — {total_trades} trades in {duration_min:.0f}min", - f"Net Position (max ${sim['position_usdt'].abs().max():,.0f})", - "Spread (bps) + Trade Signals", - ], - ) - - # Row 1: PnL breakdown - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["cum_captured"], - line=dict(color=COLORS["brl"], width=1.5), - name="Captured (gross)", - fill="tozeroy", - fillcolor="rgba(63,185,80,0.08)", - ), - row=1, - col=1, - ) - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["cum_fees"], - line=dict(color=COLORS["fees"], width=1, dash="dot"), - name="Fees paid", - ), - row=1, - col=1, - ) - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["cum_pnl"], - line=dict(color=COLORS["pnl"], width=2), - name="Net PnL (realized)", - ), - row=1, - col=1, - ) - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["total_pnl"], - line=dict(color=COLORS["spread"], width=1.5, dash="dash"), - name="Total PnL (realized + unrealized)", - ), - row=1, - col=1, - ) - - # Row 2: Position - pos_colors = [ - COLORS["long"] if v >= 0 else COLORS["short"] for v in sim["position_usdt"] - ] - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["position_usdt"], - line=dict(color=COLORS["pnl"], width=1.5), - fill="tozeroy", - fillcolor="rgba(88,166,255,0.08)", - name="Position (USDT)", - showlegend=True, - ), - row=2, - col=1, - ) - fig.add_hline(y=0, line_color=COLORS["grid"], row=2, col=1) - - # Row 3: Spread + signals - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["spread_bps"], - line=dict(color=COLORS["spread"], width=1), - name="Spread (bps)", - showlegend=True, - ), - row=3, - col=1, - ) - fig.add_hline( - y=min_spread_bps, - line_dash="dash", - line_color=COLORS["brl"], - opacity=0.5, - row=3, - col=1, - ) - fig.add_hline( - y=-min_spread_bps, - line_dash="dash", - line_color=COLORS["brl"], - opacity=0.5, - row=3, - col=1, - ) - fig.add_hline(y=0, line_color=COLORS["grid"], row=3, col=1) - - # Trade markers - sells_brl = sim[sim["action"] == "sell_brl"] - sells_usdt = sim[sim["action"] == "sell_usdt"] - if not sells_brl.empty: - fig.add_trace( - go.Scatter( - x=sells_brl["ts"], - y=sells_brl["spread_bps"], - mode="markers", - marker=dict(color=COLORS["short"], size=3, symbol="triangle-down"), - name="Sell BRL", - showlegend=True, - ), - row=3, - col=1, - ) - if not sells_usdt.empty: - fig.add_trace( - go.Scatter( - x=sells_usdt["ts"], - y=sells_usdt["spread_bps"], - mode="markers", - marker=dict(color=COLORS["long"], size=3, symbol="triangle-up"), - name="Sell USDT", - showlegend=True, - ), - row=3, - col=1, - ) - - fig.update_layout( - **LAYOUT_BASE, - title=dict( - text=( - f"Quoting Strategy Sim" - f" | ${trade_size_usdt:.0f}/trade" - f" | fee: {fee_bps:.0f}bps" - f" | min: {min_spread_bps:.0f}bps" - f" | PnL: ${final_pnl:+.2f}" - f" | Pos: ${final_position:+,.0f}" - ), - font=dict(color=COLORS["text"], size=12), - x=0.5, - ), - width=1300, - height=900, - ) - fig.update_yaxes(title_text="USDT", tickformat=",.2f", row=1, col=1) - fig.update_yaxes(title_text="USDT", tickformat=",.0f", row=2, col=1) - fig.update_yaxes(title_text="bps", row=3, col=1) - for ann in fig.layout.annotations: - ann.font = dict(color=COLORS["text_dim"], size=11) - _style_axes(fig, 3) - return fig - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: - chat_id = context._chat_id - client = await get_client(chat_id, context=context) - if not client: - return RoutineResult(text="No server available") - - # Fetch 1s candles in parallel - btc_usdt, btc_brl, usdt_brl = await asyncio.gather( - _fetch_candles( - client, config.connector_name, "BTC-USDT", "1s", config.lookback_seconds - ), - _fetch_candles( - client, config.connector_name, "BTC-BRL", "1s", config.lookback_seconds - ), - _fetch_candles( - client, config.connector_name, "USDT-BRL", "1s", config.lookback_seconds - ), - ) - - for name, df in [ - ("BTC-USDT", btc_usdt), - ("BTC-BRL", btc_brl), - ("USDT-BRL", usdt_brl), - ]: - if df.empty: - return RoutineResult(text=f"No 1s data for {name}") - - # Compute spread - merged = _merge_spread(btc_usdt, btc_brl, usdt_brl) - if merged.empty: - return RoutineResult(text="No overlapping data between pairs") - - # Run strategy simulation - sim = simulate_quoting( - merged, - trade_size_usdt=config.trade_size_usdt, - fee_bps=config.fee_bps, - min_spread_bps=config.min_spread_bps, - max_position_usdt=config.max_position_usdt, - ) - - # Build charts - spread_fig = build_spread_chart(merged) - strategy_fig = build_strategy_chart( - sim, config.fee_bps, config.min_spread_bps, config.trade_size_usdt - ) - - # Summary stats - duration_min = (merged["ts"].iloc[-1] - merged["ts"].iloc[0]).total_seconds() / 60 - avg_spread = merged["spread_pct"].mean() - std_spread = merged["spread_pct"].std() - curr_spread = merged["spread_pct"].iloc[-1] - total_trades = int(sim["trade_count"].iloc[-1]) - final_captured = sim["cum_captured"].iloc[-1] - final_fees = sim["cum_fees"].iloc[-1] - final_pnl = sim["cum_pnl"].iloc[-1] - final_position = sim["position_usdt"].iloc[-1] - final_total = sim["total_pnl"].iloc[-1] - - # Spread distribution - above_thresh = (merged["spread_bps"].abs() > config.min_spread_bps).mean() * 100 - time_positive = (merged["spread_pct"] > 0).mean() * 100 - - # Annualize PnL - hours = duration_min / 60 - if hours > 0: - pnl_per_hour = final_pnl / hours - projected_daily = pnl_per_hour * 24 - projected_monthly = projected_daily * 30 - else: - pnl_per_hour = projected_daily = projected_monthly = 0 - - text = "\n".join( - [ - "BTC-BRL Arbitrage Analysis", - f"Window: {duration_min:.0f} min ({len(merged):,} ticks)", - "", - "Spread Stats:", - f" Current: {curr_spread:+.4f}% ({curr_spread*100:+.2f} bps)", - f" Average: {avg_spread:+.4f}%", - f" Std Dev: {std_spread:.4f}%", - f" Time |spread| > {config.min_spread_bps:.0f}bps: {above_thresh:.1f}%", - f" Time BRL premium: {time_positive:.1f}%", - "", - f"Strategy Sim (${config.trade_size_usdt:.0f}/trade, {config.fee_bps:.0f}bps fee, >{config.min_spread_bps:.0f}bps):", - f" Trades: {total_trades}", - f" Gross captured: ${final_captured:.2f}", - f" Fees paid: ${final_fees:.2f}", - f" Net realized PnL: ${final_pnl:+.2f}", - f" Current position: ${final_position:+,.0f}", - f" Total PnL (incl. unrealized): ${final_total:+.2f}", - "", - "Projections (if conditions persist):", - f" Per hour: ${pnl_per_hour:+.2f}", - f" Per day: ${projected_daily:+.2f}", - f" Per month: ${projected_monthly:+.2f}", - ] - ) - - # Export primary chart - img_bytes = spread_fig.to_image(format="png", width=1300, height=700, scale=2) - chart = io.BytesIO(img_bytes) - - # Report - from condor.reports import ReportBuilder - - builder = ReportBuilder("BTC-BRL Arbitrage") - builder.source("routine", "btc_brl_arbitrage").tags( - ["btc", "brl", "arbitrage", "strategy"] - ) - builder.markdown(text) - builder.plotly(spread_fig) - builder.plotly(strategy_fig) - await builder.save() - - return RoutineResult(text=text, chart_image=chart) diff --git a/agents/brigado/routines/config_analyzer.py b/agents/brigado/routines/config_analyzer.py deleted file mode 100644 index ca811cd0c..000000000 --- a/agents/brigado/routines/config_analyzer.py +++ /dev/null @@ -1,772 +0,0 @@ -"""PMM Config Analyzer — Compare controller configs side by side, highlight differences.""" - -CATEGORY = "Bot Analysis" - -import asyncio -import logging -import re -from collections import defaultdict - -import plotly.graph_objects as go -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from config_manager import get_config_manager - -logger = logging.getLogger(__name__) - -_DARK_BG = "#1a1a2e" -_DARK_PAPER = "#16213e" -_GREEN = "#00d4aa" -_RED = "#ff6b6b" -_BLUE = "#58a6ff" -_YELLOW = "#ffdd57" -_COLORS = [ - "#58a6ff", - "#00d4aa", - "#ff6b6b", - "#ffdd57", - "#c084fc", - "#fb923c", - "#38bdf8", - "#f472b6", -] - -# Parameters to always skip in comparison (noise) -SKIP_PARAMS = {"id", "controller_type", "controller_name", "manual_kill_switch"} - -# Parameters to highlight when different (key tuning params) -KEY_PARAMS = { - "buy_spreads", - "sell_spreads", - "buy_amounts_pct", - "sell_amounts_pct", - "total_amount_quote", - "stop_loss", - "take_profit", - "time_limit", - "executor_refresh_time", - "cooldown_time", - "leverage", - "dca_spreads", - "dca_amounts", - "trailing_stop", - "executor_activation_bounds", - "top_executor_refresh_time", - "position_rebalance_threshold_pct", -} - - -class Config(BaseModel): - """Analyze and compare PMM controller configs — saved vs running bots.""" - - server_name: str = Field( - default="brigado_2", - description="Server to connect to", - ) - controller_type: str = Field( - default="", - description="Filter by controller type (e.g. 'market_making'). Empty = all types.", - ) - controller_filter: str = Field( - default="pmm_mister", - description="Filter by config name or controller_name containing this string. Empty = all.", - ) - pair_filter: str = Field( - default="", - description="Filter by trading pair (e.g. 'BRL', 'BTC-BRL'). Empty = all.", - ) - running_only: bool = Field( - default=False, - description="Show only configs that are currently running in active bots.", - ) - show_common: bool = Field( - default=False, - description="Include parameters that are the same across all configs (verbose mode).", - ) - - -def _short_name(config_name: str) -> str: - """Shorten config name for display.""" - name = config_name - name = re.sub(r"^mm_", "", name) - name = re.sub(r"pmm_?mister_?", "pm_", name) - name = re.sub(r"^(binance|okx|bybit|kucoin)(_perpetual)?_", "", name) - name = re.sub(r"config[_]?(\d+)", r"c\1", name) - while "__" in name: - name = name.replace("__", "_") - return name.strip("_") or config_name[:25] - - -def _fmt_value(v) -> str: - """Format a config value for display.""" - if v is None: - return "—" - if isinstance(v, bool): - return "✓" if v else "✗" - if isinstance(v, float): - if v < 0.001: - return f"{v:.5f}" - if v < 0.01: - return f"{v:.4f}" - return f"{v:.3f}" if v < 10 else f"{v:.1f}" - if isinstance(v, list): - return ", ".join(_fmt_value(x) for x in v) - if isinstance(v, dict): - parts = [f"{k}={_fmt_value(vv)}" for k, vv in v.items()] - return " | ".join(parts) - return str(v) - - -def _normalize_value(v): - """Normalize values for comparison.""" - if v is None: - return None - if isinstance(v, str): - try: - return float(v) - except ValueError: - return v.strip().lower() - if isinstance(v, list): - return [_normalize_value(x) for x in v] - if isinstance(v, dict): - return {k: _normalize_value(vv) for k, vv in v.items()} - return v - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: - # Connect to server - client = await get_config_manager().get_client(config.server_name) - if not client: - return f"Could not connect to server '{config.server_name}'" - - # Fetch all data in parallel - all_configs_list, bots_data = await asyncio.gather( - client.controllers.list_controller_configs(), - _get_bots_data(client), - ) - - # --- Saved configs --- - saved_configs = {} - filt = config.controller_filter.lower() if config.controller_filter else "" - for cfg_meta in all_configs_list or []: - cname = cfg_meta.get("id") or cfg_meta.get("config_name", "") - ctype = cfg_meta.get("controller_type", "") - ctrl_name = cfg_meta.get("controller_name", "") - - if ( - config.controller_type - and config.controller_type.lower() not in ctype.lower() - ): - continue - if filt and filt not in cname.lower() and filt not in ctrl_name.lower(): - continue - - try: - cfg_data = await client.controllers.get_controller_config(cname) - # Also check controller_name inside the full config data - if filt and filt not in cname.lower(): - inner_ctrl = cfg_data.get("controller_name", "") - if filt not in inner_ctrl.lower(): - continue - if config.pair_filter: - pair = cfg_data.get("trading_pair", "") - if config.pair_filter.upper() not in pair.upper(): - continue - saved_configs[cname] = cfg_data - except Exception as e: - logger.warning(f"Failed to fetch config {cname}: {e}") - - # --- Running bot configs --- - running_config_ids = set() - running_configs = {} - for bot_name in bots_data or {}: - try: - bot_cfgs = await client.controllers.get_bot_controller_configs(bot_name) - if isinstance(bot_cfgs, list): - for cfg in bot_cfgs: - cid = cfg.get("id") or cfg.get("controller_id", "") - ctype = cfg.get("controller_type", "") - if ( - config.controller_type - and config.controller_type.lower() not in ctype.lower() - ): - continue - ctrl_name_inner = cfg.get("controller_name", "") - if ( - filt - and filt not in cid.lower() - and filt not in ctrl_name_inner.lower() - ): - continue - if config.pair_filter: - pair = cfg.get("trading_pair", "") - if config.pair_filter.upper() not in pair.upper(): - continue - running_configs[cid] = cfg - running_config_ids.add(cid) - except Exception as e: - logger.warning(f"Failed to fetch bot configs for {bot_name}: {e}") - - # Merge all configs with status prefix - all_configs = {} - for name, cfg in saved_configs.items(): - prefix = "🟢" if name in running_config_ids else "🔵" - all_configs[f"{prefix} {name}"] = cfg - for cid, cfg in running_configs.items(): - if cid not in saved_configs: - all_configs[f"🟢 {cid}"] = cfg - - # Filter to running only if requested - if config.running_only: - all_configs = { - k: v for k, v in all_configs.items() if k.startswith("\U0001f7e2") - } - - if not all_configs: - filters = [] - if config.controller_type: - filters.append(f"type={config.controller_type}") - if config.controller_filter: - filters.append(f"filter={config.controller_filter}") - if config.pair_filter: - filters.append(f"pair={config.pair_filter}") - return f"No configs found matching: {', '.join(filters) if filters else 'any'}" - - # --- Build comparison --- - all_params = set() - for cfg in all_configs.values(): - all_params.update(cfg.keys()) - all_params -= SKIP_PARAMS - - # Sort: key params first, then alphabetical - sorted_params = sorted(all_params, key=lambda p: (0 if p in KEY_PARAMS else 1, p)) - - # Find which params differ - diff_params = set() - for param in sorted_params: - values = [_normalize_value(cfg.get(param)) for cfg in all_configs.values()] - non_none = [v for v in values if v is not None] - if len(set(str(v) for v in non_none)) > 1: - diff_params.add(param) - - # Group configs by trading pair - pair_groups = defaultdict(dict) - for label, cfg in all_configs.items(): - pair = cfg.get("trading_pair", "unknown") - pair_groups[pair][label] = cfg - - num_running = sum(1 for l in all_configs if l.startswith("🟢")) - - # --- Build HTML report --- - from condor.reports import ReportBuilder - - builder = ReportBuilder("PMM Config Analyzer") - builder.source("routine", "config_analyzer").tags(["configs", "pmm", "analysis"]) - builder.manual_order() - - # KPIs - builder.kpi("Total Configs", str(len(all_configs))) - builder.kpi("Running", str(num_running), trend="up" if num_running else "neutral") - builder.kpi("Saved Only", str(len(all_configs) - num_running)) - builder.kpi( - "Diffs Found", - str(len(diff_params)), - delta=f"of {len(sorted_params)} params", - ) - builder.kpi("Trading Pairs", str(len(pair_groups))) - - # --- Per-pair comparison tables --- - for pair, pair_cfgs in sorted(pair_groups.items(), key=lambda x: -len(x[1])): - n_running = sum(1 for l in pair_cfgs if l.startswith("🟢")) - - labels = list(pair_cfgs.keys()) - shorts = [_short_name(l.replace("🟢 ", "").replace("🔵 ", "")) for l in labels] - - # Find pair-level diffs - pair_diff_params = set() - for param in sorted_params: - values = [_normalize_value(pair_cfgs[l].get(param)) for l in labels] - non_none = [v for v in values if v is not None] - if len(non_none) > 0 and len(set(str(v) for v in non_none)) > 1: - pair_diff_params.add(param) - - heading_md = f"## {pair} ({len(pair_cfgs)} configs, {n_running} running)" - - if len(labels) == 1: - # Single config — show full params - label = labels[0] - cfg = pair_cfgs[label] - rows = [] - for param in sorted_params: - val = cfg.get(param) - if val is not None: - is_key = "⚡ " if param in KEY_PARAMS else "" - rows.append( - { - "Parameter": f"{is_key}{param}", - "Value": _fmt_value(val), - } - ) - builder.markdown(f"{heading_md}\n\n**{label}** — single config") - if rows: - builder.table(rows) - else: - # Multiple configs — comparison view - rows = [] - for param in sorted_params: - values = [pair_cfgs[l].get(param) for l in labels] - non_none = [v for v in values if v is not None] - if not non_none: - continue - - is_diff = param in pair_diff_params - if not config.show_common and not is_diff: - continue - - prefix = ( - "🔴 " - if is_diff and param in KEY_PARAMS - else "🟡 " if is_diff else "" - ) - row = {"Parameter": f"{prefix}{param}"} - for short, val in zip(shorts, values): - row[short] = _fmt_value(val) - rows.append(row) - - note = ( - f"Showing {len(rows)} differing params. 🔴 = key, 🟡 = other." - if not config.show_common - else "" - ) - if not rows and not config.show_common: - builder.markdown( - f"{heading_md}\n\nAll parameters identical across these configs." - ) - else: - builder.markdown(f"{heading_md}\n\n{note}" if note else heading_md) - if rows: - builder.table(rows) - - # --- Cross-pair key differences summary --- - if len(pair_groups) > 1 and diff_params: - cross_sections = [] - for param in sorted_params: - if param not in diff_params or param not in KEY_PARAMS: - continue - value_groups = defaultdict(list) - for label, cfg in all_configs.items(): - val = _fmt_value(cfg.get(param)) - short = _short_name(label.replace("🟢 ", "").replace("🔵 ", "")) - value_groups[val].append(short) - if len(value_groups) <= 1: - continue - rows = [] - for val, names in sorted(value_groups.items(), key=lambda x: -len(x[1])): - rows.append( - { - "Value": val, - "Count": str(len(names)), - "Configs": ", ".join(names[:8]) - + (f" (+{len(names)-8})" if len(names) > 8 else ""), - } - ) - cross_sections.append((param, rows)) - - if cross_sections: - builder.markdown( - "## Cross-Pair Key Differences\n\nKey tuning parameters grouped by distinct values across all configs." - ) - for param, rows in cross_sections: - builder.markdown(f"**`{param}`** — {len(rows)} distinct values:") - builder.table(rows) - - # --- Charts --- - if len(all_configs) > 1: - fig_pc = _build_parallel_coords(all_configs, running_config_ids) - if fig_pc: - builder.plotly(fig_pc) - - fig = _build_spread_chart(all_configs) - if fig: - builder.plotly(fig) - - fig2 = _build_capital_chart(all_configs) - if fig2: - builder.plotly(fig2) - - await builder.save() - - # --- Concise plain text summary --- - lines = [f"**PMM Config Analyzer** — {config.server_name}", ""] - lines.append( - f"Found **{len(all_configs)}** configs ({num_running} running, {len(all_configs) - num_running} saved-only)" - ) - lines.append(f"**{len(diff_params)}** parameters differ across configs") - lines.append( - f"**{len(pair_groups)}** pairs: {', '.join(sorted(pair_groups.keys()))}" - ) - lines.append("") - - for pair, pair_cfgs in sorted(pair_groups.items(), key=lambda x: -len(x[1])): - n_r = sum(1 for l in pair_cfgs if l.startswith("🟢")) - lines.append(f"- **{pair}**: {len(pair_cfgs)} configs ({n_r} running)") - - lines.append("") - lines.append("Full comparison report available in the dashboard.") - - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -async def _get_bots_data(client) -> dict: - try: - data = await client.bot_orchestration.get_active_bots_status() - if isinstance(data, dict) and "data" in data: - return data["data"] - except Exception: - pass - return {} - - -def _build_parallel_coords(all_configs: dict, running_ids: set) -> go.Figure | None: - """Parallel coordinates plot (HiPlot-style) for numeric config parameters.""" - # Extract numeric axes from key params - NUMERIC_AXES = [ - ("total_amount_quote", "Capital", None), - ( - "buy_spreads", - "Buy Spread L1 %", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "sell_spreads", - "Sell Spread L1 %", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "buy_spreads", - "Buy Spread L2 %", - lambda v: ( - (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None - ), - ), - ( - "sell_spreads", - "Sell Spread L2 %", - lambda v: ( - (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None - ), - ), - ("executor_refresh_time", "Refresh Time", None), - ("cooldown_time", "Cooldown", None), - ("leverage", "Leverage", None), - ("stop_loss", "Stop Loss", None), - ("take_profit", "Take Profit", None), - ("time_limit", "Time Limit", None), - ("top_executor_refresh_time", "Top Refresh", None), - ( - "dca_spreads", - "DCA Spread L1", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "dca_amounts", - "DCA Amt L1", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "trailing_stop", - "Trail Stop", - lambda v: ( - (v.get("activation_price_delta") if isinstance(v, dict) else v) - if v - else None - ), - ), - ( - "executor_activation_bounds", - "Activation Bounds", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ] - - # Collect data rows - rows = [] - labels = [] - for label, cfg in all_configs.items(): - clean_label = label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", "") - is_running = clean_label in running_ids or label.startswith("\U0001f7e2") - pair = cfg.get("trading_pair", "unknown") - row = {"_running": 1 if is_running else 0, "_pair": pair} - for param, axis_name, extractor in NUMERIC_AXES: - raw = cfg.get(param) - if extractor: - val = extractor(raw) - else: - val = raw - try: - row[axis_name] = float(val) if val is not None else None - except (TypeError, ValueError): - row[axis_name] = None - rows.append(row) - labels.append(_short_name(clean_label)) - - if len(rows) < 2: - return None - - # Filter axes that have at least 2 distinct non-None values - valid_axes = [] - for _, axis_name, _ in NUMERIC_AXES: - vals = [r[axis_name] for r in rows if r[axis_name] is not None] - if len(vals) >= 2 and len(set(vals)) > 1: - valid_axes.append(axis_name) - - if len(valid_axes) < 2: - return None - - # Build color by pair - unique_pairs = sorted(set(r["_pair"] for r in rows)) - pair_to_idx = {p: i for i, p in enumerate(unique_pairs)} - color_values = [pair_to_idx[r["_pair"]] for r in rows] - - # Build colorscale from our palette - n_pairs = len(unique_pairs) - if n_pairs == 1: - colorscale = [[0, _BLUE], [1, _BLUE]] - else: - colorscale = [] - for i in range(n_pairs): - pos = i / (n_pairs - 1) - colorscale.append([pos, _COLORS[i % len(_COLORS)]]) - - # Fill None values with axis min (so lines don't break) - for axis_name in valid_axes: - vals = [r[axis_name] for r in rows if r[axis_name] is not None] - fill = min(vals) if vals else 0 - for r in rows: - if r[axis_name] is None: - r[axis_name] = fill - - # Build dimensions - dimensions = [] - for axis_name in valid_axes: - vals = [r[axis_name] for r in rows] - # Multiply spread values by 100 for readability - if "%" in axis_name: - vals = [v * 100 for v in vals] - dimensions.append( - dict( - label=axis_name, - values=vals, - range=( - [min(vals), max(vals)] - if min(vals) != max(vals) - else [min(vals) - 1, max(vals) + 1] - ), - ) - ) - - fig = go.Figure( - data=go.Parcoords( - line=dict( - color=color_values, - colorscale=colorscale, - showscale=False, - ), - dimensions=dimensions, - customdata=labels, - ) - ) - - # Add pair legend via invisible scatter traces - for i, pair in enumerate(unique_pairs): - fig.add_trace( - go.Scatter( - x=[None], - y=[None], - mode="markers", - marker=dict(size=10, color=_COLORS[i % len(_COLORS)]), - name=pair, - showlegend=True, - ) - ) - - fig.update_layout( - title=dict( - text=f"Config Parameter Space ({len(rows)} configs, {len(valid_axes)} axes)", - font=dict(color="#e6edf3", size=14), - x=0.5, - ), - paper_bgcolor=_DARK_PAPER, - plot_bgcolor=_DARK_BG, - font=dict(color="#ccc", size=11), - margin=dict(l=80, r=80, t=80, b=40), - width=1400, - height=600, - legend=dict( - orientation="h", y=-0.05, x=0.5, xanchor="center", font=dict(size=12) - ), - ) - return fig - - -def _build_spread_chart(all_configs: dict) -> go.Figure | None: - """Grouped bar chart comparing buy/sell spreads across configs.""" - - def _to_list(v) -> list: - if v is None: - return [] - if isinstance(v, (int, float)): - return [float(v)] - if isinstance(v, list): - return [float(x) for x in v] - return [] - - configs_with_spreads = [] - for label, cfg in all_configs.items(): - buy = _to_list(cfg.get("buy_spreads")) - sell = _to_list(cfg.get("sell_spreads")) - if buy or sell: - short = _short_name(label.replace("🟢 ", "").replace("🔵 ", "")) - pair = cfg.get("trading_pair", "") - configs_with_spreads.append( - { - "name": short, - "pair": pair, - "buy_l1": buy[0] * 100 if len(buy) >= 1 else 0, - "sell_l1": sell[0] * 100 if len(sell) >= 1 else 0, - "buy_l2": buy[1] * 100 if len(buy) >= 2 else 0, - "sell_l2": sell[1] * 100 if len(sell) >= 2 else 0, - } - ) - - if not configs_with_spreads: - return None - - configs_with_spreads.sort(key=lambda c: (c["pair"], c["name"])) - names = [c["name"] for c in configs_with_spreads] - - fig = go.Figure() - fig.add_trace( - go.Bar( - x=names, - y=[c["buy_l1"] for c in configs_with_spreads], - name="Buy L1", - marker_color=_GREEN, - opacity=0.9, - ) - ) - fig.add_trace( - go.Bar( - x=names, - y=[c["sell_l1"] for c in configs_with_spreads], - name="Sell L1", - marker_color=_RED, - opacity=0.9, - ) - ) - - has_l2 = any(c["buy_l2"] > 0 or c["sell_l2"] > 0 for c in configs_with_spreads) - if has_l2: - fig.add_trace( - go.Bar( - x=names, - y=[c["buy_l2"] for c in configs_with_spreads], - name="Buy L2", - marker_color=_GREEN, - opacity=0.5, - ) - ) - fig.add_trace( - go.Bar( - x=names, - y=[c["sell_l2"] for c in configs_with_spreads], - name="Sell L2", - marker_color=_RED, - opacity=0.5, - ) - ) - - fig.update_layout( - title=dict( - text="Spread Levels by Config (%)", - font=dict(color="#e6edf3", size=14), - x=0.5, - ), - paper_bgcolor=_DARK_PAPER, - plot_bgcolor=_DARK_BG, - font=dict(color="#ccc", size=10), - barmode="group", - yaxis=dict(gridcolor="#2a2a4a", title="Spread %"), - xaxis=dict(gridcolor="#2a2a4a", tickangle=-45), - legend=dict(orientation="h", y=1.1, x=0.5, xanchor="center"), - margin=dict(l=60, r=40, t=60, b=120), - width=1200, - height=500, - ) - return fig - - -def _build_capital_chart(all_configs: dict) -> go.Figure | None: - """Horizontal bar chart: capital allocation per config, colored by pair.""" - data = [] - for label, cfg in all_configs.items(): - amt = float(cfg.get("total_amount_quote", 0) or 0) - if amt <= 0: - continue - short = _short_name(label.replace("🟢 ", "").replace("🔵 ", "")) - pair = cfg.get("trading_pair", "") - is_running = label.startswith("🟢") - data.append({"name": short, "pair": pair, "amount": amt, "running": is_running}) - - if not data: - return None - - data.sort(key=lambda d: (d["pair"], d["amount"])) - - unique_pairs = list(dict.fromkeys(d["pair"] for d in data)) - pair_colors = {p: _COLORS[i % len(_COLORS)] for i, p in enumerate(unique_pairs)} - - fig = go.Figure() - for pair in unique_pairs: - pair_data = [d for d in data if d["pair"] == pair] - fig.add_trace( - go.Bar( - y=[d["name"] for d in pair_data], - x=[d["amount"] for d in pair_data], - orientation="h", - name=pair, - marker_color=pair_colors[pair], - opacity=0.85, - text=[ - f"{'🟢' if d['running'] else ''} {d['amount']:,.0f}" - for d in pair_data - ], - textposition="auto", - ) - ) - - height = max(400, len(data) * 22 + 100) - fig.update_layout( - title=dict( - text="Capital Allocation by Config (quote currency)", - font=dict(color="#e6edf3", size=14), - x=0.5, - ), - paper_bgcolor=_DARK_PAPER, - plot_bgcolor=_DARK_BG, - font=dict(color="#ccc", size=10), - barmode="stack", - xaxis=dict(gridcolor="#2a2a4a", title="Amount (quote)"), - yaxis=dict(gridcolor="#2a2a4a"), - legend=dict(orientation="h", y=1.05, x=0.5, xanchor="center"), - margin=dict(l=150, r=40, t=60, b=50), - width=1200, - height=height, - ) - return fig diff --git a/agents/brigado/routines/spot_markets.py b/agents/brigado/routines/spot_markets.py deleted file mode 100644 index a3795b995..000000000 --- a/agents/brigado/routines/spot_markets.py +++ /dev/null @@ -1,458 +0,0 @@ -"""Spot market discovery — top markets by volume across quote currencies with rebate focus.""" - -CATEGORY = "Market Data" - -import asyncio -import logging -from typing import Any - -import aiohttp -import pandas as pd -import plotly.graph_objects as go -from plotly.subplots import make_subplots -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from config_manager import get_client -from routines.base import RoutineResult - -logger = logging.getLogger(__name__) - -BINANCE_SPOT_TICKER = "https://api.binance.com/api/v3/ticker/24hr" - -# Quote currencies we care about -REBATE_QUOTES = { - "BRL", - "ARS", - "MXN", - "EUR", - "PLN", - "JPY", - "TRY", - "RON", - "UAH", - "ZAR", - "NGN", - "COP", -} -STABLE_QUOTES = {"USDT", "USDC", "BUSD", "FDUSD", "TUSD"} - -# Stablecoins / wrapped tokens to exclude as base -EXCLUDED_BASES = {"USDT", "USDC", "BUSD", "TUSD", "FDUSD", "DAI", "BFUSD"} - -COLORS = { - "bg_paper": "#0d1117", - "bg_plot": "#161b22", - "grid": "#21262d", - "text": "#c9d1d9", - "text_dim": "#8b949e", - "green": "#3fb950", - "red": "#f85149", - "blue": "#58a6ff", - "yellow": "#f0b90b", - "purple": "#bc8cff", - "orange": "#f78166", -} - -QUOTE_COLORS = { - "BRL": "#3fb950", - "USDT": "#e6e6e6", - "FDUSD": "#58a6ff", - "USDC": "#2775ca", - "ARS": "#75b5f0", - "MXN": "#f0b90b", - "EUR": "#f78166", - "PLN": "#bc8cff", - "JPY": "#ff7b72", - "TRY": "#d2a8ff", - "RON": "#7ee787", - "UAH": "#79c0ff", - "ZAR": "#ffa657", - "NGN": "#56d364", - "COP": "#e3b341", -} - - -class Config(BaseModel): - """Spot market discovery — top pairs by volume, FIAT rebate markets, and USDT leaders.""" - - top_n: int = Field(default=50, description="Top N markets per section") - min_volume_usd: float = Field( - default=100_000, description="Min 24h volume (USD) for FIAT pairs" - ) - min_volume_usdt: float = Field( - default=5_000_000, description="Min 24h volume (USD) for USDT pairs" - ) - focus_quote: str = Field( - default="BRL", description="Primary FIAT quote to highlight" - ) - - -def _parse_quote(symbol: str, known_quotes: set[str]) -> tuple[str, str] | None: - """Parse a Binance symbol into (base, quote) using known quote list.""" - for q in sorted(known_quotes, key=len, reverse=True): - if symbol.endswith(q): - base = symbol[: -len(q)] - if base: - return base, q - return None - - -async def fetch_spot_tickers() -> list[dict]: - """Fetch all 24h tickers from Binance spot.""" - async with aiohttp.ClientSession() as session: - async with session.get(BINANCE_SPOT_TICKER) as resp: - resp.raise_for_status() - return await resp.json() - - -def categorize_tickers(tickers: list[dict]) -> dict[str, list[dict]]: - """Parse and categorize tickers by quote currency.""" - all_quotes = REBATE_QUOTES | STABLE_QUOTES | {"BTC", "ETH", "BNB"} - by_quote: dict[str, list[dict]] = {} - - for t in tickers: - symbol = t.get("symbol", "") - parsed = _parse_quote(symbol, all_quotes) - if not parsed: - continue - base, quote = parsed - if base in EXCLUDED_BASES: - continue - - quote_vol = float(t.get("quoteVolume", 0)) - if quote_vol <= 0: - continue - - entry = { - "symbol": symbol, - "base": base, - "quote": quote, - "trading_pair": f"{base}-{quote}", - "price": float(t.get("lastPrice", 0)), - "price_change_pct": float(t.get("priceChangePercent", 0)), - "volume_24h": float(t.get("volume", 0)), - "quote_volume_24h": quote_vol, - "trades_24h": int(t.get("count", 0)), - } - by_quote.setdefault(quote, []).append(entry) - - # Sort each group by quote volume - for q in by_quote: - by_quote[q].sort(key=lambda x: x["quote_volume_24h"], reverse=True) - - return by_quote - - -async def _fetch_usdt_rates(tickers: list[dict]) -> dict[str, float]: - """Extract USDT rates for fiat currencies from ticker data.""" - rates = {"USDT": 1.0, "USDC": 1.0, "FDUSD": 1.0, "BUSD": 1.0, "TUSD": 1.0} - # BTC-USDT price for cross-conversion - btc_usdt = 0.0 - for t in tickers: - sym = t.get("symbol", "") - price = float(t.get("lastPrice", 0)) - if sym == "BTCUSDT": - btc_usdt = price - # Direct USDT/FIAT pairs (e.g., USDTBRL → 1 USDT = X BRL) - if sym.startswith("USDT") and price > 0: - fiat = sym[4:] - if fiat in REBATE_QUOTES: - rates[fiat] = 1.0 / price # BRL per USDT → USDT per BRL - # Also check FIAT/USDT pairs (unlikely on Binance but just in case) - for t in tickers: - sym = t.get("symbol", "") - price = float(t.get("lastPrice", 0)) - if sym.endswith("USDT") and price > 0: - base = sym[:-4] - if base in REBATE_QUOTES and base not in rates: - rates[base] = price - return rates - - -def format_volume(v: float) -> str: - if v >= 1_000_000_000: - return f"${v / 1_000_000_000:.1f}B" - if v >= 1_000_000: - return f"${v / 1_000_000:.1f}M" - if v >= 1_000: - return f"${v / 1_000:.0f}K" - return f"${v:.0f}" - - -def _build_report( - by_quote: dict[str, list[dict]], - usdt_rates: dict[str, float], - config: Config, -) -> tuple[str, go.Figure, list[dict]]: - """Build text summary, chart, and table data.""" - - # --- Section 1: Focus FIAT quote (BRL by default) --- - focus = config.focus_quote.upper() - focus_pairs = by_quote.get(focus, []) - focus_rate = usdt_rates.get(focus, 0) - - # --- Section 2: All FIAT markets summary --- - fiat_summary = [] - for q in sorted(REBATE_QUOTES): - pairs = by_quote.get(q, []) - if not pairs: - continue - rate = usdt_rates.get(q, 0) - total_vol_usdt = ( - sum(p["quote_volume_24h"] * rate for p in pairs) if rate > 0 else 0 - ) - fiat_summary.append( - { - "quote": q, - "pair_count": len(pairs), - "total_volume_usdt": total_vol_usdt, - "top_pair": pairs[0]["trading_pair"] if pairs else "-", - "top_volume": ( - pairs[0]["quote_volume_24h"] * rate if pairs and rate > 0 else 0 - ), - } - ) - fiat_summary.sort(key=lambda x: x["total_volume_usdt"], reverse=True) - - # --- Section 3: Top USDT pairs --- - usdt_pairs = [ - p - for p in by_quote.get("USDT", []) - if p["quote_volume_24h"] >= config.min_volume_usdt - ] - usdt_top = usdt_pairs[: config.top_n] - - # --- Build text --- - lines = [] - - # FIAT overview - lines.append("FIAT MARKETS OVERVIEW (Rebate Eligible)") - lines.append("-" * 45) - total_fiat_vol = sum(f["total_volume_usdt"] for f in fiat_summary) - lines.append(f"Total FIAT volume: {format_volume(total_fiat_vol)} USDT equiv.") - lines.append("") - for f in fiat_summary: - lines.append( - f" {f['quote']:>4s}: {f['pair_count']:3d} pairs | " - f"Vol: {format_volume(f['total_volume_usdt']):>8s} | " - f"Top: {f['top_pair']}" - ) - - # Focus FIAT detail - if focus_pairs and focus_rate > 0: - filtered = [ - p - for p in focus_pairs - if p["quote_volume_24h"] * focus_rate >= config.min_volume_usd - ] - lines.append("") - lines.append(f"TOP {focus}-QUOTED MARKETS (Rebates)") - lines.append("-" * 45) - for i, p in enumerate(filtered[: config.top_n], 1): - vol_usdt = p["quote_volume_24h"] * focus_rate - chg = p["price_change_pct"] - chg_s = f"+{chg:.1f}%" if chg >= 0 else f"{chg:.1f}%" - lines.append( - f" {i:2d}. {p['trading_pair']:<12s} " - f"Vol: {format_volume(vol_usdt):>8s} " - f"24h: {chg_s:>7s} " - f"Trades: {p['trades_24h']:,}" - ) - - # Top USDT - lines.append("") - lines.append(f"TOP {config.top_n} USDT-QUOTED MARKETS") - lines.append("-" * 45) - for i, p in enumerate(usdt_top, 1): - chg = p["price_change_pct"] - chg_s = f"+{chg:.1f}%" if chg >= 0 else f"{chg:.1f}%" - lines.append( - f" {i:2d}. {p['trading_pair']:<12s} " - f"Vol: {format_volume(p['quote_volume_24h']):>8s} " - f"24h: {chg_s:>7s} " - f"Trades: {p['trades_24h']:,}" - ) - - text = "\n".join(lines) - - # --- Chart: two panels --- - fig = make_subplots( - rows=2, - cols=1, - subplot_titles=[ - f"FIAT Market Volume (USDT equiv.)", - f"Top 30 USDT Markets — Volume vs 24h Change", - ], - row_heights=[0.35, 0.65], - vertical_spacing=0.12, - ) - - # Panel 1: FIAT volume bar chart - if fiat_summary: - quotes = [f["quote"] for f in fiat_summary] - volumes = [f["total_volume_usdt"] for f in fiat_summary] - bar_colors = [QUOTE_COLORS.get(q, "#8b949e") for q in quotes] - fig.add_trace( - go.Bar( - x=quotes, - y=volumes, - marker_color=bar_colors, - text=[format_volume(v) for v in volumes], - textposition="outside", - textfont=dict(size=9, color=COLORS["text"]), - hovertemplate="%{x}: %{y:,.0f} USDT", - showlegend=False, - ), - row=1, - col=1, - ) - - # Panel 2: Scatter — volume vs price change for top USDT - scatter_data = usdt_top[:30] - if scatter_data: - volumes_s = [p["quote_volume_24h"] for p in scatter_data] - changes = [p["price_change_pct"] for p in scatter_data] - labels = [p["base"] for p in scatter_data] - scatter_colors = [COLORS["green"] if c >= 0 else COLORS["red"] for c in changes] - - fig.add_trace( - go.Scatter( - x=changes, - y=volumes_s, - mode="markers+text", - marker=dict( - color=scatter_colors, - size=[max(8, min(25, v / max(volumes_s) * 25)) for v in volumes_s], - line=dict(width=1, color=COLORS["bg_paper"]), - ), - text=labels, - textposition="top center", - textfont=dict(size=8, color=COLORS["text_dim"]), - hovertemplate="%{text}
Vol: %{y:,.0f}
Chg: %{x:.1f}%", - showlegend=False, - ), - row=2, - col=1, - ) - - fig.update_layout( - paper_bgcolor=COLORS["bg_paper"], - plot_bgcolor=COLORS["bg_plot"], - font=dict(color=COLORS["text_dim"], size=10), - margin=dict(l=60, r=30, t=60, b=40), - height=800, - ) - for i in range(1, 3): - fig.update_xaxes(gridcolor=COLORS["grid"], row=i, col=1) - fig.update_yaxes(gridcolor=COLORS["grid"], row=i, col=1) - fig.update_yaxes(type="log", title="Volume (USDT)", row=1, col=1) - fig.update_yaxes(type="log", title="Volume 24h (USDT)", row=2, col=1) - fig.update_xaxes(title="24h Change (%)", row=2, col=1) - - # Annotation styling for subplot titles - for ann in fig.layout.annotations: - ann.font = dict(size=12, color=COLORS["text"]) - - # --- Table data for report --- - table_rows = [] - - # FIAT summary table - for f in fiat_summary: - table_rows.append( - { - "Section": "FIAT Overview", - "Quote": f["quote"], - "Pairs": f["pair_count"], - "Total Volume": format_volume(f["total_volume_usdt"]), - "Top Pair": f["top_pair"], - } - ) - - # Focus pairs table - if focus_pairs and focus_rate > 0: - for p in focus_pairs[: config.top_n]: - vol_usdt = p["quote_volume_24h"] * focus_rate - if vol_usdt < config.min_volume_usd: - continue - table_rows.append( - { - "Section": f"{focus} Markets", - "Pair": p["trading_pair"], - "Volume (USDT)": format_volume(vol_usdt), - "24h Change": f"{p['price_change_pct']:+.1f}%", - "Trades": f"{p['trades_24h']:,}", - } - ) - - # USDT top pairs - for p in usdt_top[: config.top_n]: - table_rows.append( - { - "Section": "USDT Markets", - "Pair": p["trading_pair"], - "Volume (USDT)": format_volume(p["quote_volume_24h"]), - "24h Change": f"{p['price_change_pct']:+.1f}%", - "Trades": f"{p['trades_24h']:,}", - } - ) - - return text, fig, table_rows - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: - """Discover spot markets across quote currencies.""" - # Fetch all tickers - try: - tickers = await fetch_spot_tickers() - except Exception as e: - return RoutineResult(text=f"Failed to fetch Binance spot tickers: {e}") - - if not tickers: - return RoutineResult(text="No ticker data returned from Binance.") - - # Categorize and get rates - by_quote = categorize_tickers(tickers) - usdt_rates = await _fetch_usdt_rates(tickers) - - # Build output - text, fig, table_rows = _build_report(by_quote, usdt_rates, config) - - # Generate report - from condor.reports import ReportBuilder - - # Split tables by section - fiat_overview = [r for r in table_rows if r.get("Section") == "FIAT Overview"] - focus_rows = [ - r - for r in table_rows - if r.get("Section") == f"{config.focus_quote.upper()} Markets" - ] - usdt_rows = [r for r in table_rows if r.get("Section") == "USDT Markets"] - - # Clean section column - for rows in [fiat_overview, focus_rows, usdt_rows]: - for r in rows: - r.pop("Section", None) - - builder = ReportBuilder("Spot Market Discovery") - builder.source("routine", "spot_markets").tags( - ["spot", "markets", "discovery", "fiat", "rebates"] - ) - builder.plotly(fig) - - builder.markdown("### FIAT Markets Overview (Rebate Eligible)") - if fiat_overview: - builder.table(fiat_overview) - - if focus_rows: - builder.markdown(f"### Top {config.focus_quote.upper()}-Quoted Markets") - builder.table(focus_rows) - - builder.markdown("### Top USDT-Quoted Markets") - if usdt_rows: - builder.table(usdt_rows) - - await builder.save() - - return RoutineResult(text=text) diff --git a/agents/brigado/routines/toxic_flow_detector.py b/agents/brigado/routines/toxic_flow_detector.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/agents/brigado/strategies/brl_mm/strategy.md b/agents/brigado/strategies/brl_mm/strategy.md deleted file mode 100644 index ca1ffb6e1..000000000 --- a/agents/brigado/strategies/brl_mm/strategy.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: BRL MM -description: BRL market making research agent. Tracks USDT-BRL spreads, BTC/fiat volume - dynamics, and bot performance for the Brazilian market making operation. -agent_key: ollama:qwen3.5:9 -skills: [] -default_config: - connector_name: binance - quote_currency: BRL -default_trading_context: '' -created_by: 481175164 -created_at: '2026-05-04T00:00:00+00:00' ---- - -### Routines - -#### brl_dashboard -Multi-timeframe technical dashboard for USDT-BRL with candlesticks, SMA, RSI, and support/resistance levels across 4 timeframes (1m, 15m, 1h, 1d). - -#### btc_brl_volume -Compares BTC-USDT vs BTC-BRL volume over 7 days and analyzes 1-second price arbitrage spread to detect opportunities. - -#### btc_fiat_markets -Stacked area chart of BTC trading volume across fiat pairs (BRL, ARS, MXN, EUR, JPY, PLN), normalized to USDT. - -#### bot_report -Comprehensive bot performance report with volume, PnL, fees, market share, rebates, and per-controller breakdown. - -### Journal -- Log market observations: BRL premium/discount trends, volume shifts -- Track bot performance metrics over time -- Note arbitrage windows and their duration diff --git a/condor/agents/README.md b/condor/agents/README.md index 33e7ca69e..9a132acc1 100644 --- a/condor/agents/README.md +++ b/condor/agents/README.md @@ -151,7 +151,7 @@ The name is resolved once at engine start and written back into the session's `c - The **base rules at the top of the prompt switch with it** (`BASE_PROMPT_LIVE_CONTROLLER` vs `BASE_PROMPT_LIVE_EXECUTORS` in `condor/agents/prompts.py`): controller mode names controllers as the trading surface, demotes `manage_executors` to an opt-in fallback, preloads `manage_bots` + `manage_controllers` via ToolSearch, and drops the executor-only `controller_id` hint. A single shared base prompt used to open every tick with "Trade ONLY via `manage_executors`" and then contradict itself in the `[CONTROLLER MODE]` block. - The agent's reported PnL becomes `executor_pnl(controller_id == agent_id)` **+** `bot_pnl(bot_name == config.bot_name)`. The two sources are disjoint — bot controllers tag their executors with their own config ids, never the `agent_id` — so the merge is plain addition with no double counting (`condor/fetchers/bot_performance.py` aggregates the bot side; `condor/agents/performance.py` folds it in). -**Ownership is a namespace, enforced at the tool call** (FEAT-017, `condor/agents/ownership.py`). Every bot a strategy may deploy or mutate lives under `{agent_slug}-{strategy_slug}` — the base itself, any tagged sibling (`…-btc`, `…-eth`), and their deployed instances (Hummingbot appends a `-YYYYMMDD-HHMMSS` timestamp). Slugs never contain `-` (`strategy._slugify` maps `[\s-]+ → _`), so the delimiter is unambiguous and two agents can never claim the same bot — uniqueness is structural, with no registry or lock. +**Ownership is a namespace, enforced at the tool call** (FEAT-017, `condor/agents/ownership.py`). Every bot a strategy may deploy or mutate lives under `{agent_slug}-{strategy_slug}` — the base itself, any tagged sibling (`…-btc`, `…-eth`), and their deployed instances (Hummingbot appends a `-YYYYMMDD-HHMMSS` timestamp). Slugs never contain `-` (`condor.frontmatter.slugify` maps `[\s-]+ → _`), so the delimiter is unambiguous and two agents can never claim the same bot — uniqueness is structural, with no registry or lock. - A `manage_bots` call with a *dangerous* action (`deploy`, `stop_bot`, `start_controllers`, `stop_controllers`, `update_config`) on a bot outside the namespace is **cancelled** at the permission callback (`condor/agents/risk.py:auto_approve_with_risk_check`) and recorded. Read-only actions (`status`, `logs`, `get_config`) are never restricted — an agent still sees the whole fleet. - A `bot_name` configured before this convention (e.g. `river-scalper`) is carried as a **declared** name and stays owned, so existing agents keep working with no config change. diff --git a/condor/agents/__init__.py b/condor/agents/__init__.py index 8ef29d838..36731d813 100644 --- a/condor/agents/__init__.py +++ b/condor/agents/__init__.py @@ -15,7 +15,7 @@ save_full_config, ) from .engine import TickEngine -from .journal import JournalManager, get_session_dir, next_session_number +from .journal import JournalManager, next_session_number from .risk import RiskEngine, RiskLimits from .strategy import Strategy, StrategyStore, split_key @@ -30,7 +30,6 @@ "save_full_config", "TickEngine", "JournalManager", - "get_session_dir", "next_session_number", "RiskEngine", "RiskLimits", diff --git a/condor/agents/agent.py b/condor/agents/agent.py index 4a748e4ff..e72a0bbcd 100644 --- a/condor/agents/agent.py +++ b/condor/agents/agent.py @@ -46,10 +46,9 @@ from datetime import datetime, timezone from pathlib import Path +from condor.frontmatter import parse_frontmatter, render_frontmatter, slugify +from condor.fsutil import atomic_write_text from condor.memory.paths import CHAT_SLUG -from condor.memory.store import _parse_frontmatter - -from .strategy import _render_frontmatter, _slugify log = logging.getLogger(__name__) @@ -160,7 +159,7 @@ def _load_agent_from_dir(agent_dir: Path) -> Agent | None: if not path.exists(): return None try: - meta, body = _parse_frontmatter(path.read_text()) + meta, body = parse_frontmatter(path.read_text()) return Agent( slug=agent_dir.name, name=meta.get("name", agent_dir.name), @@ -254,7 +253,7 @@ def create( server_name: str = "", created_by: int = 0, ) -> Agent: - slug = _slugify(name) + slug = slugify(name) if slug == CHAT_SLUG: # Reserved: `condor` names the default agent. The separate trees used # to make the collision impossible by construction (FEAT-003); with @@ -316,8 +315,9 @@ def _save(self, agent: Agent) -> None: "created_at": agent.created_at, } agent.agent_dir.mkdir(parents=True, exist_ok=True) - (agent.agent_dir / "AGENT.md").write_text( - _render_frontmatter(meta, agent.instructions) + atomic_write_text( + agent.agent_dir / "AGENT.md", + render_frontmatter(meta, agent.instructions), ) def _iter_agent_dirs(self): diff --git a/condor/agents/attribution.py b/condor/agents/attribution.py new file mode 100644 index 000000000..bc97a78c2 --- /dev/null +++ b/condor/agents/attribution.py @@ -0,0 +1,292 @@ +"""Session-ownership PnL attribution engine ([[FEAT-018]], [[ARCH-191]]). + +Single source of truth for how a bot's realized PnL / volume / trades / fees are +attributed to the sessions that operated it. The vocabulary lives here in one +place: + +- **session ownership resolution** — which bases a session owned and since when, + from the ledger ``owned_bots.json`` writes (:mod:`condor.agents.ownership`) + with a legacy shim for sessions predating it; +- **owner-window tiling** — the per-base ``[since_i, since_{i+1})`` windows that + reproduce a bot's whole cumulative with no gap and no double count; +- **slice-and-merge** — folding a sliced history window into a performance + object, including the shared fee rules (``fees_known`` heuristic and the + live ``cum_fees_quote`` fallback), which exist exactly once, here; +- **the current-owner rule** — live unrealized PnL and the open book belong to + whoever operates the bot now. + +Both consumers go through this module: the web strategy rollup +(``condor.web.routes.agents``) via :func:`apply_bot_mode_pnl` / +:func:`current_owner_bases` / :func:`session_ownership`, and the agent's own +view (``condor.agents.performance``) via :func:`fold_sliced_window` / +:func:`sliced_or_live_fees`. That is what makes the invariant structural +instead of hand-maintained: the dashboard and the tick loop cannot disagree, +because they no longer have separate copies of the rules to drift apart. +""" + +from __future__ import annotations + +import logging +import os +import time +from pathlib import Path +from typing import Any + +from condor.agents.ownership import OwnedBot, read_owned +from condor.agents.sessions_index import find_session_dir + +log = logging.getLogger(__name__) + + +# ── Shared merge rules ── +# ``perf`` is duck-typed over the attribute vocabulary the two consumers share: +# the agents-side ``AgentPerformance`` dataclass and the web's +# ``AgentPerformanceModel`` both carry realized_pnl / unrealized_pnl / total_pnl / +# volume / fees / fees_known / trade_count / closed_count. + + +def fold_sliced_window(perf: Any, window: tuple[float, float, float, float]) -> None: + """Fold one sliced ``(realized, volume, trades, fees)`` window into ``perf``. + + Sliced closes are round-trip closes, so they are this window's trades AND + its closed positions — the same two counters bumped on both surfaces, so the + session detail and the rollup report one number. + """ + realized, volume, trades, fees = window + closes = int(round(trades)) + perf.realized_pnl += realized + perf.volume += volume + perf.fees += fees + # Traded but charged nothing: the fee column is missing, not the fees. + if volume > 0 and fees == 0.0: + perf.fees_known = False + perf.trade_count += closes + perf.closed_count += closes + perf.total_pnl = perf.realized_pnl + perf.unrealized_pnl + + +def sliced_or_live_fees(sliced_fees: float, bot: dict[str, Any]) -> float: + """Fees from the sliced history, else the live snapshot's cumulative figure. + + A backend that reports no cumulative fee column slices to zero; the live + open-position figure is then the only one there is — attributed rather than + silently dropped. + """ + if sliced_fees: + return sliced_fees + return float(bot.get("cum_fees_quote", 0) or 0) + + +# ── Session ownership resolution ── + + +def session_bot_base(strategy_dir: Path, default_config: dict | None, num: int) -> str: + """Bot base name a session operates: per-session config, else strategy default. + + A non-empty per-session ``bot_name`` wins (so runtime-named bots that record + their deployed name resolve), but an empty/absent one falls back to the + strategy default — early sessions predating the config's ``bot_name`` saved it + as ``''`` and must still map to the shared bot they operated. Empty string when + neither is set (direct-executor strategies). Shared by the per-session PnL + distribution and the operator's live-executor view so both resolve identically. + """ + from condor.agents.config import load_full_config + + default_base = (default_config or {}).get("bot_name", "") or "" + sd = find_session_dir(strategy_dir, num) + if not sd: + return default_base + return load_full_config(sd, default_config).get("bot_name", "") or default_base + + +def session_start_epoch(strategy_dir: Path, num: int) -> float: + """Session start time: config.yml is written once at start, so its mtime is stable.""" + sd = find_session_dir(strategy_dir, num) + if not sd: + return 0.0 + cfg = sd / "config.yml" + target = cfg if cfg.exists() else sd + try: + return os.path.getmtime(target) + except OSError: + return 0.0 + + +def session_ownership( + strategy_dir: Path, default_config: dict | None, num: int +) -> list[OwnedBot]: + """Bases a session owned and the instant it took each over, oldest first. + + Two sources, in order: + + 1. ``{session_dir}/owned_bots.json`` — the ledger [[FEAT-017]] writes, which + knows both the bases (a session may operate several) and the exact takeover + instant, whether the bot was deployed here or adopted after a restart. + 2. the legacy shim — a session predating the ledger resolves its single + ``bot_name`` as one owned bot ``since`` the session started, reproducing the + session-start tiling attribution used before the ledger existed. + + Empty for direct-executor strategies, whose per-session executor attribution + already stands and must not be touched. + """ + owned = read_owned(find_session_dir(strategy_dir, num)) + if owned: + return owned + base = session_bot_base(strategy_dir, default_config, num) + if not base: + return [] + start = session_start_epoch(strategy_dir, num) + return [OwnedBot(base=base, origin="legacy", since=start, last_seen=start)] + + +# ── Owner-window tiling ── + + +def owner_windows( + real_sessions: list, strategy_dir: Path, default_config: dict | None +) -> dict[str, list[tuple[float, Any, float]]]: + """``{base: [(since, session, until), …]}`` — owners, oldest takeover first. + + The windows a base's owners occupy tile ``[since_i, since_{i+1})`` and the last + one runs to now, so slicing over them reproduces the bot's whole cumulative with + no gap and no double count. Keyed per base rather than globally per session + number: two bases handed over at different moments never share a timeline. + + ``until`` is the instant the session released the bot, or ``0.0`` while it + still holds it. A released last window stops there rather than running to now, + which is the one case where the tiling deliberately leaves a gap: PnL a bot + earned with no session operating it belongs to no session. + """ + owners: dict[str, list[tuple[float, Any, float]]] = {} + for s in real_sessions: + for ob in session_ownership(strategy_dir, default_config, s.session_num): + owners.setdefault(ob.base, []).append((ob.since, s, ob.until)) + for lst in owners.values(): + lst.sort(key=lambda t: (t[0], t[1].session_num)) + return owners + + +def current_owner_bases( + strategy_dir: Path, + default_config: dict | None, + session_nums: list[int], + num: int, +) -> list[str]: + """Bases ``num`` is the CURRENT owner of — the last takeover by ``since``. + + A bot's live open positions belong to whoever operates it now, so this is the + gate for merging them into one session's view. Same rule + :func:`apply_bot_mode_pnl` applies to live unrealized PnL, kept here as one + lookup over the same windows so the rollup and the per-session detail can + never disagree about who holds the open book. A session that released the bot + is not its current owner, so an ended session shows no live open book. + """ + last: dict[str, tuple[float, int, float]] = {} + for n in session_nums: + for ob in session_ownership(strategy_dir, default_config, n): + if last.get(ob.base, (float("-inf"), -1, 0.0))[:2] <= (ob.since, n): + last[ob.base] = (ob.since, n, ob.until) + return sorted( + base for base, (_, owner, until) in last.items() if owner == num and until <= 0 + ) + + +# ── The attribution engine ── + + +async def apply_bot_mode_pnl( + real_sessions: list, strategy_dir: Path, default_config: dict | None, client: Any +) -> None: + """Distribute each owned bot's PnL across the sessions that operated it. + + One rule covers deploy and handover: every owned bot is attributed by slicing + its history over ``[since, next_owner.since or now)``, where ``since`` is the + takeover instant the ownership ledger recorded. A bot the session *deployed* + has no history before its ``since``, so the general rule already hands it the + whole instance — the exact case falls out instead of needing its own branch. + + Live unrealized PnL, fees and open positions go to each base's LAST owner by + ``since`` — a lookup in the ledger where it used to be a ``max(session_num)`` + guess, so a new session that never adopted the bot no longer inherits its + open book. + + Works uniformly for single- and multi-controller bots (history sums controllers + per instance) and for a base re-launched under several instances. Strategies + whose sessions own no bot (direct-executor agents) are left untouched. + """ + from condor.fetchers.bot_performance import ( + bot_executor_rows, + fetch_all_bot_performance, + fetch_archived_instances, + fetch_base_histories, + resolve_bots, + slice_history, + ) + + if not client or not real_sessions: + return + owners = owner_windows(real_sessions, strategy_dir, default_config) + bases = sorted(owners) + if not bases: + return # direct-executor strategy — nothing to attribute + + try: + all_perf = await fetch_all_bot_performance(client) + except Exception as e: + log.warning("bot perf fetch for %s failed: %s", strategy_dir.name, e) + return + + now = time.time() + # The oldest takeover across every base sets how far back the histories must + # reach; sampling resolution is chosen from it so no owner's window falls off + # the end of the retained rows. + earliest = min( + (since for lst in owners.values() for since, _, _ in lst if since > 0), + default=0.0, + ) + # Archived instances carry the realized PnL of every bot a session stopped — + # the normal end state of a finished session, and invisible in the live + # snapshot. Same universe the live agent's own view uses, so the dashboard and + # the tick loop cannot disagree about what a session earned. + archived = await fetch_archived_instances(client) + histories_by_base = await fetch_base_histories( + client, all_perf, bases, earliest, now, extra_names=archived + ) + + live = resolve_bots(all_perf, bases) + for base in bases: + window_owners = owners[base] + insts = histories_by_base.get(base, []) + + # Realized / volume / trades / fees: one window per owner, tiling the + # timeline. A released window (the session stopped and left the bot + # running) ends at its release instant, so PnL earned while no session + # was operating the bot is attributed to nobody instead of accruing to + # whoever happened to hold last. + sliced_fees = 0.0 + for i, (since, s, until) in enumerate(window_owners): + end = window_owners[i + 1][0] if i + 1 < len(window_owners) else now + if until > 0: + end = min(end, until) + if end <= since: + continue + window = slice_history(insts, since, end) + fold_sliced_window(s, window) + sliced_fees += window[3] + + # Live unrealized + open positions → the base's current owner, unless it + # has released the bot: an ended session holds no open book. + bot = live.get(base) + if not bot: + continue + last_since, operator, last_until = window_owners[-1] + if last_until > 0: + continue + b_rows = bot_executor_rows(bot) + operator.unrealized_pnl += float(bot.get("unrealized_pnl_quote", 0) or 0) + # Top the operator up to the fallback figure: when the whole sliced fee + # column is zero, the live open-position fees are the only ones there are. + operator.fees += sliced_or_live_fees(sliced_fees, bot) - sliced_fees + operator.open_count += sum(1 for r in b_rows if r["status"] == "RUNNING") + operator.executors = list(operator.executors) + b_rows + operator.total_pnl = operator.realized_pnl + operator.unrealized_pnl diff --git a/condor/agents/config.py b/condor/agents/config.py index 88b418acd..1af7f89ff 100644 --- a/condor/agents/config.py +++ b/condor/agents/config.py @@ -12,6 +12,8 @@ import yaml from pydantic import BaseModel, Field +from condor.fsutil import atomic_write_text + class RiskLimitsConfig(BaseModel): max_position_size_quote: float = Field( @@ -124,8 +126,9 @@ def save_agent_config(agent_dir: Path, config: AgentConfig) -> None: """Save config to config.yml in the agent directory.""" config_path = agent_dir / "config.yml" agent_dir.mkdir(parents=True, exist_ok=True) - config_path.write_text( - yaml.dump(config.model_dump(), default_flow_style=False, sort_keys=False) + atomic_write_text( + config_path, + yaml.dump(config.model_dump(), default_flow_style=False, sort_keys=False), ) @@ -160,4 +163,6 @@ def save_full_config(agent_dir: Path, config: dict[str, Any]) -> None: """Save a raw config dict as YAML (no filtering through AgentConfig).""" config_path = agent_dir / "config.yml" agent_dir.mkdir(parents=True, exist_ok=True) - config_path.write_text(yaml.dump(config, default_flow_style=False, sort_keys=False)) + atomic_write_text( + config_path, yaml.dump(config, default_flow_style=False, sort_keys=False) + ) diff --git a/condor/agents/consult.py b/condor/agents/consult.py index 5f8eab5e3..2a7b0620b 100644 --- a/condor/agents/consult.py +++ b/condor/agents/consult.py @@ -21,12 +21,14 @@ import os from condor.acp.pydantic_ai_client import ( - PydanticAIClient, healthcheck_local_backend, is_pydantic_ai_model, ) from condor.agents.agent import AgentStore from condor.preferences import resolve_custom_endpoint +from condor.runtime import context as runtime_context +from condor.runtime import toolsets +from condor.runtime.channels import TelegramChannel log = logging.getLogger(__name__) @@ -43,7 +45,6 @@ def _build_consult_permission_cb(slug: str, user_id: int, chat_id: int): try: from condor.routine_store import get_routine_store from condor.runtime.confirmations import build_permission_callback - from handlers.agents.confirmation import TelegramChannel bot = get_routine_store().get_bot() if bot is not None: @@ -150,9 +151,6 @@ async def _run_agent_to_completion( fallback, ) model_key = fallback - # The fallback is a different model — re-resolve, or a custom - # endpoint's credentials would leak into an unrelated backend. - base_url, api_key = resolve_custom_endpoint(model_key, user_id=user_id) fallback_note = ( f"_(note: {agent.name}'s configured model was unavailable — " f"{backend_err} Answered with fallback `{fallback}`.)_\n\n" @@ -166,12 +164,6 @@ async def _run_agent_to_completion( # Build the Agent's MCP toolset in the main process (ConfigManager is here). # agent_slug scopes the condor MCP tools' memory/skills to this Agent (its brain). - from handlers.agents._shared import ( - build_agent_context, - build_mcp_servers_for_session, - get_project_dir, - ) - # A server pinned on the Agent itself wins over the ambient chat server; when # the agent isn't pinned, fall back to the caller's (chat's) resolved server. # Passing server_name=None lets the builder resolve the chat's server. @@ -179,7 +171,7 @@ async def _run_agent_to_completion( # agent_slug the condor MCP tools would target the CHAT's stores. effective_server = agent.server_name or server_name - mcp_servers = build_mcp_servers_for_session( + mcp_servers = toolsets.build_mcp_servers_for_session( user_id, chat_id, server_name=effective_server if agent.server_required else None, @@ -192,33 +184,24 @@ async def _run_agent_to_completion( # agent auto-approves (unattended). permission_cb = permission_callback - # Build the client for the (possibly fallback) model. A pydantic-ai model gets - # the agent's tool allowlist enforced; an ACP fallback (claude-code) cannot - # enforce an allowlist, so it runs the consult unrestricted — acceptable since - # it is the trusted coordinator model and mutations are still confirmation-gated. - if is_pydantic_ai_model(model_key): - client = PydanticAIClient( - model=model_key, - mcp_servers=mcp_servers, - permission_callback=permission_cb, - allowed_tools=agent.tools or None, - base_url=base_url, - api_key=api_key, - ) - else: - from condor.acp.client import ACPClient, resolve_acp - - agent_cmd, model_env, model_pref = resolve_acp(model_key) - client = ACPClient( - command=agent_cmd, - working_dir=get_project_dir(), - mcp_servers=mcp_servers, - permission_callback=permission_cb, - extra_env=model_env or None, - model=model_pref or None, - ) + # Build the client for the (possibly fallback) model through the shared + # factory (ARCH-192). A pydantic-ai model gets the agent's tool allowlist + # enforced; an ACP fallback (claude-code) cannot enforce an allowlist, so it + # runs the consult unrestricted — acceptable since it is the trusted + # coordinator model and mutations are still confirmation-gated. The factory + # re-resolves the custom endpoint (same lenient inputs as the healthcheck + # above), so a fallback model never inherits the original's credentials. + from condor.runtime.llm_client import build_llm_client + + client = build_llm_client( + model_key, + mcp_servers=mcp_servers, + permission_callback=permission_cb, + allowed_tools=agent.tools or None, + user_id=user_id, + ) - prompt = build_agent_context(agent, user_id, task, context) + prompt = runtime_context.build_agent_context(agent, user_id, task, context) await client.start() try: diff --git a/condor/agents/delegate.py b/condor/agents/delegate.py index 618e90245..e43292b8c 100644 --- a/condor/agents/delegate.py +++ b/condor/agents/delegate.py @@ -689,11 +689,14 @@ def resolve_bot(bot=None): """The best Telegram sender available in this process. Prefer the passed live ``bot``; otherwise fall back to the registered routine - bot, and finally the ``_HttpBot`` Telegram-HTTP path (``TELEGRAM_TOKEN``) that + bot, then the ``_HttpBot`` Telegram-HTTP path (``TELEGRAM_TOKEN``) that routines/notification already use, so a process with no live bot still - delivers. Public because a Telegram wake sink (FEAT-034) must reach the user - through exactly the same ladder a completion notice does -- two ladders would - eventually disagree about who can talk to a chat. + delivers. With no token at all the last rung is ``NotifyBot`` (FEAT-048), + which records the message as a dashboard notification -- an install with no + Telegram now tells the user things instead of dropping them. Public because a + Telegram wake sink (FEAT-034) must reach the user through exactly the same + ladder a completion notice does -- two ladders would eventually disagree + about who can talk to a chat. """ target = bot if target is None: @@ -706,7 +709,16 @@ def resolve_bot(bot=None): if target is None: from condor.routine_store import _HttpBot - target = _HttpBot() + http = _HttpBot() + if http.has_token(): + return http + # No Telegram at all: record on the dashboard instead of handing the + # message to a sender that drops it (FEAT-048). This rung sits *below* + # both Telegram rungs, so an install with a bot is byte-for-byte + # unchanged and nothing is ever delivered twice to Telegram. + from condor.notifications import NotifyBot + + return NotifyBot() return target @@ -715,4 +727,16 @@ async def _notify_done(dt: DelegateTask, bot) -> None: if not dt.chat_id: return - await resolve_bot(bot).send_message(chat_id=dt.chat_id, text=_completion_text(dt)) + from condor.notifications import NotifyBot, record + + text = _completion_text(dt) + target = resolve_bot(bot) + await target.send_message(chat_id=dt.chat_id, text=text) + + # And on the dashboard bell (FEAT-048), with the same text the Telegram push + # and the transcript note carry, so the three surfaces cannot tell three + # stories about one task. Skipped when the resolved sender *is* the bell: + # that rung only wins when there is no Telegram, and it has already recorded + # this very message. + if not isinstance(target, NotifyBot): + await record(dt.user_id, text, kind="delegation") diff --git a/condor/agents/engine.py b/condor/agents/engine.py index 5be524798..94a75d0fc 100644 --- a/condor/agents/engine.py +++ b/condor/agents/engine.py @@ -24,9 +24,9 @@ ToolCallEvent, ToolCallUpdate, fold_tool_call_event, - resolve_acp, ) -from condor.acp.pydantic_ai_client import PydanticAIClient, is_pydantic_ai_model +from condor.acp.pydantic_ai_client import PydanticAIClient +from condor.runtime import toolsets from condor.runtime.registry_file import LoopState from condor.runtime.timeouts import resolve_tick_timeout from condor.telemetry import taps as telemetry_taps @@ -204,6 +204,7 @@ def __post_init__(self): self.strategy.slug, self.session_num, frequency_sec=self.config.get("frequency_sec", 60), + owner_id=self.user_id, ) risk_limits = RiskLimits.from_dict(self.config.get("risk_limits", {})) @@ -865,15 +866,10 @@ async def _create_client( (it only feeds the auto-approve callback and cannot change between the two points), avoiding a redundant per-tick journal re-parse. """ - from handlers.agents._shared import ( - build_mcp_servers_for_session, - get_project_dir, - ) - mode = self.config.get("execution_mode", "loop") # A configured server pins the toolset; None falls back to the chat's. - mcp_servers = build_mcp_servers_for_session( + mcp_servers = toolsets.build_mcp_servers_for_session( self.user_id, self.chat_id, server_name=self.config.get("server_name"), @@ -887,47 +883,21 @@ async def _create_client( agent_id=self.agent_id, ) - agent_key = self._agent_key() - use_pydantic_ai = is_pydantic_ai_model(agent_key) - - if use_pydantic_ai: - import os - - from condor.preferences import resolve_custom_endpoint - - # A custom endpoint's URL/key come from the owner's saved endpoints; - # an explicit model_base_url in the run config still wins. - custom_url, api_key = resolve_custom_endpoint( - agent_key, user_id=self.user_id - ) - base_url = self.config.get("model_base_url") or custom_url or None - tool_filter_mode = ( - self.config.get("tool_filter_mode") - or os.environ.get("PYDANTIC_AI_TOOL_FILTER") - or None - ) - return PydanticAIClient( - model=agent_key, - mcp_servers=mcp_servers, - permission_callback=permission_cb, - base_url=base_url, - api_key=api_key, - tool_filter_mode=tool_filter_mode, - # Same allowlist the agent gets on consult; empty => unrestricted. - allowed_tools=self.agent.tools or None, - ) - else: - # Supports a Claude model suffix, e.g. "claude-acp:opus" — selected via - # session/set_model after handshake (the bridge ignores ANTHROPIC_MODEL). - agent_cmd, model_env, model_pref = resolve_acp(agent_key) - return ACPClient( - command=agent_cmd, - working_dir=get_project_dir(), - mcp_servers=mcp_servers, - permission_callback=permission_cb, - extra_env=model_env or None, - model=model_pref or None, - ) + # Shared factory (ARCH-192). Engine specifics: an explicit model_base_url + # in the run config still wins over the owner's saved custom endpoint, + # and the run config's tool_filter_mode beats the env fallback. Same + # allowlist the agent gets on consult; empty => unrestricted. + from condor.runtime.llm_client import build_llm_client + + return build_llm_client( + self._agent_key(), + mcp_servers=mcp_servers, + permission_callback=permission_cb, + allowed_tools=self.agent.tools or None, + user_id=self.user_id, + base_url_override=self.config.get("model_base_url") or None, + tool_filter_mode=self.config.get("tool_filter_mode"), + ) # ------------------------------------------------------------------ # Helpers diff --git a/condor/agents/journal.py b/condor/agents/journal.py index d7a943add..8240c0560 100644 --- a/condor/agents/journal.py +++ b/condor/agents/journal.py @@ -166,15 +166,6 @@ def resolve_agent_dirs(agent_id: str) -> tuple[Path | None, Path | None]: """ -def get_session_dir(run_key: str, session_number: int) -> Path: - """Build the path for a specific session directory. - - ``run_key`` is the ``"{agent_slug}.{strategy_slug}"`` prefix (legacy flat - slugs without a dot still resolve to ``agents/{slug}/``). - """ - return _strategy_base_dir(run_key) / "sessions" / f"session_{session_number}" - - def next_session_number(agent_dir: Path) -> int: """Determine the next session number by scanning existing session_* dirs.""" # Check new location first @@ -400,13 +391,13 @@ def __init__( self._archive_last_section: str | None = None if not self._path.exists(): - self._path.write_text(JOURNAL_TEMPLATE.format(agent_id=agent_id)) + atomic_write_text(self._path, JOURNAL_TEMPLATE.format(agent_id=agent_id)) # Ensure learnings.md exists at agent level if self._agent_dir: learnings_path = self._agent_dir / "learnings.md" if not learnings_path.exists(): - learnings_path.write_text(LEARNINGS_TEMPLATE) + atomic_write_text(learnings_path, LEARNINGS_TEMPLATE) self._tick_count = self._count_ticks() @@ -466,7 +457,7 @@ def append_learning(self, text_content: str, category: str = "") -> None: return if not path.exists(): - path.write_text(LEARNINGS_TEMPLATE) + atomic_write_text(path, LEARNINGS_TEMPLATE) # Resolve category to section header cat = category if category in LEARNING_CATEGORIES else DEFAULT_LEARNING_CATEGORY @@ -518,7 +509,7 @@ def append_learning(self, text_content: str, category: str = "") -> None: ) else: new_text = full_text.rstrip() + f"\n\n## {section_header}\n{new_section}\n" - path.write_text(new_text) + atomic_write_text(path, new_text) def _append_learning_to_journal(self, text_content: str) -> None: """Legacy: append learning to journal's Learnings section.""" diff --git a/condor/agents/ownership.py b/condor/agents/ownership.py index 913ed2ab7..1ffb33427 100644 --- a/condor/agents/ownership.py +++ b/condor/agents/ownership.py @@ -3,7 +3,7 @@ An agent running in controller mode owns the bots it deploys, and the **name is the proof**: every bot a given ``(agent, strategy)`` may touch lives under the namespace ``{agent_slug}-{strategy_slug}``. Slugs never contain ``-`` -(``strategy._slugify`` maps ``[\\s-]+ → _``), so ``-`` delimits the namespace +(``condor.frontmatter.slugify`` maps ``[\\s-]+ → _``), so ``-`` delimits the namespace unambiguously and two different agents can never claim the same bot — uniqueness is structural, with no registry, lock or lookup. @@ -28,6 +28,8 @@ from pathlib import Path from typing import Any, Iterable +from condor.fsutil import atomic_write_json + log = logging.getLogger(__name__) LEDGER_FILENAME = "owned_bots.json" @@ -347,9 +349,6 @@ def _save(self) -> None: if not path: return # experiments: in-memory only try: - path.parent.mkdir(parents=True, exist_ok=True) - tmp = path.with_suffix(".json.tmp") - tmp.write_text(json.dumps(self.to_dict(), indent=2)) - tmp.replace(path) + atomic_write_json(path, self.to_dict(), indent=2) except Exception: log.warning("BotLedger: could not write %s", path, exc_info=True) diff --git a/condor/agents/performance.py b/condor/agents/performance.py index f50e97a73..149415f5c 100644 --- a/condor/agents/performance.py +++ b/condor/agents/performance.py @@ -167,40 +167,35 @@ def _merge_bot_perf( the bot snapshot reports how many positions closed but not how each one ended, so a bot-mode agent has no per-trade outcome to derive one from. """ + from condor.agents.attribution import fold_sliced_window, sliced_or_live_fees from condor.fetchers.bot_performance import bot_executor_rows rows = bot_executor_rows(bot) open_rows = [r for r in rows if r["status"] == "RUNNING"] if window is None: - closes = int(bot.get("closed_trades", 0) or 0) - volume_added = float(bot.get("volume_traded", 0) or 0) - fees_added = float(bot.get("cum_fees_quote", 0) or 0) - perf.realized_pnl += float(bot.get("realized_pnl_quote", 0) or 0) - perf.volume += volume_added - perf.fees += fees_added + # Lifetime aggregate, folded through the same shared rule as a window so + # the fees_known heuristic exists exactly once (condor.agents.attribution). + fold_sliced_window( + perf, + ( + float(bot.get("realized_pnl_quote", 0) or 0), + float(bot.get("volume_traded", 0) or 0), + float(bot.get("closed_trades", 0) or 0), + float(bot.get("cum_fees_quote", 0) or 0), + ), + ) else: realized, volume, trades, fees = window - closes = int(round(trades)) - perf.realized_pnl += realized - perf.volume += volume - # A backend that reports no cumulative fee column slices to zero; the - # live open-position figure is then the only one there is. - fees_added = fees if fees else float(bot.get("cum_fees_quote", 0) or 0) - volume_added = volume - perf.fees += fees_added - - # Traded but charged nothing: the fee column is missing, not the fees. - if volume_added > 0 and fees_added == 0.0: - perf.fees_known = False + fold_sliced_window( + perf, (realized, volume, trades, sliced_or_live_fees(fees, bot)) + ) for ct, n in (bot.get("close_type_counts") or {}).items(): perf.close_type_counts[str(ct)] = perf.close_type_counts.get(str(ct), 0) + int( n or 0 ) - perf.trade_count += closes - perf.closed_count += closes perf.unrealized_pnl += float(bot.get("unrealized_pnl_quote", 0) or 0) perf.total_pnl = perf.realized_pnl + perf.unrealized_pnl perf.controllers = perf.controllers + list(bot.get("controllers", [])) @@ -238,7 +233,7 @@ async def fetch_agent_pnl_series( fetch_all_bot_performance, fetch_archived_instances, fetch_base_histories, - slice_history, + slice_history_series, ) bases = [b for b in (bot_names or []) if b] @@ -267,10 +262,13 @@ async def fetch_agent_pnl_series( # One point per instant any instance was sampled. Each is the whole session's # cumulative at that moment, so the curve is continuous across a redeploy # rather than restarting at zero with each new bot. + # Single merge pass over all instances instead of a slice_history rescan + # per stamp — same values, O(stamps + rows) instead of O(stamps × rows). stamps = sorted({t for h in instances for t, *_ in h if since <= t <= end}) series: list[dict[str, Any]] = [] - for t in stamps: - realized, volume, _trades, _fees = slice_history(instances, since, t) + for t, realized, volume, _trades, _fees in slice_history_series( + instances, since, stamps + ): series.append( { "timestamp": datetime.fromtimestamp(t, tz=timezone.utc).isoformat(), @@ -294,16 +292,9 @@ def _merge_stopped_bot_perf( the normal way a session ends. Everything an open position would contribute (unrealized PnL, executor rows) is correctly absent. """ - realized, volume, trades, fees = window - closes = int(round(trades)) - perf.realized_pnl += realized - perf.volume += volume - perf.fees += fees - if volume > 0 and fees == 0.0: - perf.fees_known = False - perf.trade_count += closes - perf.closed_count += closes - perf.total_pnl = perf.realized_pnl + perf.unrealized_pnl + from condor.agents.attribution import fold_sliced_window + + fold_sliced_window(perf, window) def _build_perf_from_rows( diff --git a/condor/agents/risk.py b/condor/agents/risk.py index 39018f900..07124931b 100644 --- a/condor/agents/risk.py +++ b/condor/agents/risk.py @@ -11,6 +11,13 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any +from condor.runtime.danger import ( + DANGEROUS_BOT_ACTIONS, + is_dangerous_tool_call, + tool_call_input, + tool_call_name, +) + if TYPE_CHECKING: from .ownership import BotLedger @@ -136,8 +143,6 @@ def check_executor_action( Returns (allowed, reason). """ - from handlers.agents._shared import tool_call_input - input_data = tool_call_input(tool_call) if input_data is None: return False, "Tool arguments could not be read" @@ -186,8 +191,6 @@ def check_bot_action(self, tool_call: dict) -> tuple[bool, str]: Returns (allowed, reason). """ - from handlers.agents._shared import tool_call_input - input_data = tool_call_input(tool_call) if input_data is None: return False, "Tool arguments could not be read" @@ -240,12 +243,6 @@ def auto_approve_with_risk_check( mistyped one open a live position no session can ever claim. Empty (consults, chat, tests) keeps the presence-only check. """ - from handlers.agents._shared import ( - DANGEROUS_BOT_ACTIONS, - is_dangerous_tool_call, - tool_call_input, - tool_call_name, - ) async def callback(tool_call: dict, options: list[dict]) -> dict: if is_dangerous_tool_call(tool_call): diff --git a/condor/agents/session_report.py b/condor/agents/session_report.py index 1fda39daa..8c2d64fb5 100644 --- a/condor/agents/session_report.py +++ b/condor/agents/session_report.py @@ -28,6 +28,7 @@ from pathlib import Path from typing import Any +from condor.reports import store from condor.reports.builder import LiveReport from . import canvas as canvas_mod @@ -47,9 +48,13 @@ def __init__( strategy_slug: str, session_num: int, frequency_sec: int = 60, + owner_id: int = 0, ) -> None: self.run_key = f"{agent_slug}.{strategy_slug}" self.session_num = session_num + # The engine's authenticated user_id: stamped on the report at save so + # the web reports routes can authorize reads against it (SEC-196). + self._owner_id = owner_id self._live = LiveReport( title=f"{agent_slug} · {strategy_slug} · session {session_num}", source_name=f"{self.run_key}/session_{session_num}", @@ -86,7 +91,8 @@ async def update( self._decisions(b, journal) self._belief_history(b, session_dir) - return await self._live.update() + with store.attribute_owner(self._owner_id): + return await self._live.update() # ------------------------------------------------------------------ # Blocks diff --git a/condor/agents/shutdown.py b/condor/agents/shutdown.py index a6ed6095a..03f8c8886 100644 --- a/condor/agents/shutdown.py +++ b/condor/agents/shutdown.py @@ -20,9 +20,10 @@ import logging from typing import Any +from condor.frontmatter import parse_frontmatter from condor.runtime.timeouts import resolve_tick_timeout -from .strategy import Strategy, _parse_frontmatter +from .strategy import Strategy log = logging.getLogger(__name__) @@ -93,7 +94,7 @@ def load_shutdown_policy(strategy: Strategy) -> tuple[ShutdownPolicy, str]: if not path.exists(): continue try: - meta, body = _parse_frontmatter(path.read_text()) + meta, body = parse_frontmatter(path.read_text()) return ShutdownPolicy.from_dict(meta), body.strip() except Exception: log.exception("Failed to parse shutdown.md at %s", path) diff --git a/condor/agents/strategy.py b/condor/agents/strategy.py index 2b60e4a99..80efc94de 100644 --- a/condor/agents/strategy.py +++ b/condor/agents/strategy.py @@ -27,14 +27,14 @@ from __future__ import annotations import logging -import re import shutil from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from typing import Any -import yaml +from condor.frontmatter import parse_frontmatter, render_frontmatter, slugify +from condor.fsutil import atomic_write_text log = logging.getLogger(__name__) @@ -62,47 +62,6 @@ """ -def _slugify(name: str) -> str: - """Convert a name to a filesystem-safe slug. - - Example: "RIVER Scalper v2" -> "river_scalper_v2" - """ - s = name.lower().strip() - s = re.sub(r"[^\w\s-]", "", s) - s = re.sub(r"[\s-]+", "_", s) - return s.strip("_") or "unnamed" - - -def _parse_frontmatter(text: str) -> tuple[dict, str]: - """Parse YAML frontmatter and markdown body from a file.""" - text = text.strip() - if not text.startswith("---"): - return {}, text - - end = text.find("---", 3) - if end == -1: - return {}, text - - frontmatter_str = text[3:end].strip() - body = text[end + 3 :].strip() - - try: - meta = yaml.safe_load(frontmatter_str) or {} - except yaml.YAMLError: - log.warning("Failed to parse YAML frontmatter") - meta = {} - - return meta, body - - -def _render_frontmatter(meta: dict, body: str) -> str: - """Render YAML frontmatter + markdown body.""" - frontmatter = yaml.dump( - meta, default_flow_style=False, allow_unicode=True, sort_keys=False - ).strip() - return f"---\n{frontmatter}\n---\n\n{body}\n" - - @dataclass class Strategy: agent_slug: str # the owning Agent's slug @@ -123,7 +82,7 @@ def __post_init__(self): @property def slug(self) -> str: """Filesystem-safe slug derived from the strategy name (unique per agent).""" - return _slugify(self.name) + return slugify(self.name) @property def key(self) -> str: @@ -139,7 +98,7 @@ def dir(self) -> Path: def split_key(key: str) -> tuple[str, str] | None: """Split an opaque strategy key ``"{agent_slug}.{slug}"`` into its parts. - Slugs never contain ``.`` (``_slugify`` strips it), so the first dot is the + Slugs never contain ``.`` (:func:`condor.frontmatter.slugify` strips it), so the first dot is the boundary. Returns None when the key has no dot. """ if "." not in key: @@ -151,7 +110,7 @@ def split_key(key: str) -> tuple[str, str] | None: def _load_strategy_from_file(path: Path, agent_slug: str) -> Strategy | None: """Load a Strategy from a ``strategy.md`` file under an agent.""" try: - meta, body = _parse_frontmatter(path.read_text()) + meta, body = parse_frontmatter(path.read_text()) return Strategy( agent_slug=agent_slug, name=meta.get("name", path.parent.name), @@ -324,6 +283,7 @@ def _save(self, strategy: Strategy) -> None: "created_at": strategy.created_at, } strategy.dir.mkdir(parents=True, exist_ok=True) - self._strategy_md_path(strategy).write_text( - _render_frontmatter(meta, strategy.instructions) + atomic_write_text( + self._strategy_md_path(strategy), + render_frontmatter(meta, strategy.instructions), ) diff --git a/condor/code_runner.py b/condor/code_runner.py index 752843c85..0e28298be 100644 --- a/condor/code_runner.py +++ b/condor/code_runner.py @@ -124,18 +124,31 @@ def _render_result(namespace: dict) -> str: async def _run_in_task( - compiled, namespace: dict, buffer, agent: str, source: str, out: dict + compiled, + namespace: dict, + buffer, + agent: str, + source: str, + out: dict, + owner_id: int = 0, ): """The snippet itself, in its own task context. Setting the capture ContextVar *here* (not in the caller) is what keeps the buffer private to this run, and the report id is read in a ``finally`` so a snippet that saved a report and then raised still reports it. + ``owner_id`` is the authenticated caller (``chat_id`` in ``execute_code``); + it stamps any report the snippet saves so the web routes can authorize + reads/deletes against it (SEC-196). """ _CAPTURE.set(buffer) reports.reset_last_report_id() try: - with reports.attribute_to(agent), reports.default_source("code", source): + with ( + reports.attribute_owner(owner_id), + reports.attribute_to(agent), + reports.default_source("code", source), + ): outcome = eval(compiled, namespace) # noqa: S307 - this IS the feature if inspect.isawaitable(outcome): await outcome @@ -207,7 +220,15 @@ async def execute_code( _install_stdout_proxy() out: dict = {"report_id": ""} task = asyncio.create_task( - _run_in_task(compiled, namespace, buffer, agent, label or run_id, out) + _run_in_task( + compiled, + namespace, + buffer, + agent, + label or run_id, + out, + owner_id=chat_id, + ) ) try: await asyncio.wait_for(task, timeout=timeout) diff --git a/condor/controller_configs.py b/condor/controller_configs.py new file mode 100644 index 000000000..c5bae6876 --- /dev/null +++ b/condor/controller_configs.py @@ -0,0 +1,51 @@ +"""Controller-config normalization shared by every surface (ARCH-190). + +Pure helpers for cleaning a controller config before it is saved to the +backend. Moved here from ``handlers/bots/_shared.py`` so the web dashboard +(``condor/web/routes/bots.py``) no longer imports from the Telegram handlers +package; that module re-exports these names for its Telegram callers. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict + +# Matches stringified enum values like "PositionMode.ONEWAY", "OrderType.LIMIT", +# "TradeType.BUY" or the colon variant "PositionMode:ONEWAY". The class name is +# PascalCase and the member starts with an uppercase letter, which avoids matching +# legitimate values such as trading times ("09:30") or token symbols ("USDC.e"). +_ENUM_STR_RE = re.compile(r"^[A-Z][A-Za-z0-9_]*[.:][A-Z][A-Za-z0-9_]*$") + + +def normalize_enum_value(value: Any) -> Any: + """Strip the enum-class prefix from stringified enum values. + + Controller config templates expose enum defaults as strings like + "PositionMode.ONEWAY". Saving them verbatim breaks downstream controller + validation, which expects the plain member name ("ONEWAY"). This recurses + into dicts and lists so nested config values are normalized too. + """ + if isinstance(value, str) and _ENUM_STR_RE.match(value): + return re.split(r"[.:]", value, maxsplit=1)[1] + if isinstance(value, dict): + return {k: normalize_enum_value(v) for k, v in value.items()} + if isinstance(value, list): + return [normalize_enum_value(v) for v in value] + return value + + +def clean_config_for_save(config: Dict[str, Any]) -> Dict[str, Any]: + """Strip internal fields and normalize stringified enum values before saving. + + The backend adds _config_name to YAML files, but Pydantic models with + extra='forbid' reject it on reload. Strip any underscore-prefixed keys + that aren't part of the controller config schema. + + Enum fields (e.g. position_mode, side, take_profit_order_type) can arrive as + prefixed strings like "PositionMode.ONEWAY"; normalize them to the plain + member name so deploy/backtest validation accepts them. + """ + return { + k: normalize_enum_value(v) for k, v in config.items() if not k.startswith("_") + } diff --git a/condor/dex_candles.py b/condor/dex_candles.py index 86bed8c57..4113df163 100644 --- a/condor/dex_candles.py +++ b/condor/dex_candles.py @@ -32,10 +32,6 @@ # apart from a plain ticker like "BTC". ADDRESS_RE = re.compile(r"^(0x[0-9a-fA-F]{40}|[1-9A-HJ-NP-Za-km-z]{32,44})$") -# GeckoTerminal answers are shared across users, so one process-wide dict backs the -# OHLCV cache for every dashboard viewer. -_ohlcv_cache: dict = {} - def uses_gecko_candles(connector: str) -> bool: """Whether this connector's candles come from GeckoTerminal, not the API.""" @@ -72,9 +68,10 @@ async def _pool_ohlcv( # Prices in the quote token, matching the scale of the entry and range # prices the executor overlays draw on the same chart. currency="token", - # A live poll wants the newest candle, and the shared cache holds a - # pool for five minutes — long enough to freeze a streaming chart. - user_data=_ohlcv_cache if use_cache else None, + # A live poll wants the newest candle, and pool_data's shared cache + # holds a pool for five minutes — long enough to freeze a streaming + # chart — so use_cache=False bypasses the cached copy. + use_cache=use_cache, limit=limit, before_timestamp=before_timestamp, token=token, diff --git a/condor/fetchers/__init__.py b/condor/fetchers/__init__.py index e305cf52c..9d4ff7927 100644 --- a/condor/fetchers/__init__.py +++ b/condor/fetchers/__init__.py @@ -21,7 +21,7 @@ above ``_SNAPSHOT_TTL``). The aggregate ``fetch_all_bot_performance`` returns is therefore shared between callers and must be treated as read-only. - - No imports from handlers/ or condor.web/ (prevents circular deps). + - No handlers/ or condor.web imports (prevents circular deps). - Keep thin: call client method, light transform, return. Importing: diff --git a/condor/fetchers/bot_performance.py b/condor/fetchers/bot_performance.py index 7417f1cc3..e9f0e32ee 100644 --- a/condor/fetchers/bot_performance.py +++ b/condor/fetchers/bot_performance.py @@ -20,7 +20,7 @@ import asyncio import logging import time -from collections import Counter +from collections import Counter, OrderedDict from functools import partial from typing import Any, Iterable @@ -475,6 +475,32 @@ def extract_history_rows(result: Any) -> list[dict]: HISTORY_PAGE_SIZE = 500 +# ── Per-instance history cache ── +# The cursor walk is the expensive end of the pipeline — up to 10 sequential +# 500-row pages per instance, fanned out over up to MAX_HISTORY_INSTANCES +# instances — and every consumer walks the same rows: one session-detail request +# triggers it twice back-to-back (performance rollup, then PnL series), and the +# tick engine repeats it every tick while the dashboard polls the same strategy. +# The rows are identical across those call sites because the fetch takes no +# since/until — window slicing happens downstream — so a short TTL plus in-flight +# coalescing (the _snapshot_cache idiom above) collapses the redundancy while +# staying fresher than the 30s route-level cache layered on top. Bounded as an +# LRU: keys accumulate one entry per (server, instance, interval) and stopped +# instances would otherwise pin their pages forever. +_HISTORY_TTL = 20.0 +_HISTORY_CACHE_MAX = 256 +_history_cache: OrderedDict[ + tuple, tuple[float, list[tuple[float, float, float, float, float]]] +] = OrderedDict() +_history_inflight: dict[tuple, tuple[Any, asyncio.Task]] = {} + + +def clear_history_cache() -> None: + """Drop every cached instance history (tests, server reconfiguration).""" + _history_cache.clear() + _history_inflight.clear() + + async def fetch_instance_history( client: Any, instance_name: str, @@ -521,7 +547,64 @@ async def fetch_instance_history( error, including one raised part-way through the walk — a partial timeline is the same silent misattribution as a truncated one, so the caller degrades to "no history" instead of to "wrong history". + + Cached per ``(server, instance, interval, limit, max_rows)`` for + ``_HISTORY_TTL`` seconds and coalesced while in flight, so concurrent callers + share one cursor walk. A walk that raises is never cached — every waiter gets + the ``[]`` degrade and the next call retries — and an unidentifiable client + (no ``base_url``) bypasses the cache entirely, like the snapshot cache above. + The returned rows are shared between callers and must be treated as + read-only. """ + server = _server_key(client) + if not server: + try: + return await _walk_instance_history( + client, instance_name, interval, limit, max_rows + ) + except Exception as e: + logger.debug("fetch_instance_history(%s) failed: %s", instance_name, e) + return [] + + key = (server, instance_name, interval, limit, max_rows) + entry = _history_cache.get(key) + if entry is not None and time.monotonic() - entry[0] <= _HISTORY_TTL: + _history_cache.move_to_end(key) + return entry[1] + + # Reuse an in-flight walk only from the loop that created it: a task is + # bound to its loop and awaiting it from another one raises. + loop = asyncio.get_running_loop() + inflight = _history_inflight.get(key) + task = inflight[1] if inflight is not None and inflight[0] is loop else None + if task is None: + task = asyncio.ensure_future( + _walk_instance_history(client, instance_name, interval, limit, max_rows) + ) + _history_inflight[key] = (loop, task) + task.add_done_callback(lambda _t, k=key: _history_inflight.pop(k, None)) + + try: + rows = await task + except Exception as e: + logger.debug("fetch_instance_history(%s) failed: %s", instance_name, e) + return [] + + _history_cache[key] = (time.monotonic(), rows) + _history_cache.move_to_end(key) + while len(_history_cache) > _HISTORY_CACHE_MAX: + _history_cache.popitem(last=False) + return rows + + +async def _walk_instance_history( + client: Any, + instance_name: str, + interval: str, + limit: int, + max_rows: int, +) -> list[tuple[float, float, float, float, float]]: + """One uncached cursor walk + forward-carry merge. Raises on API error.""" def _warn_truncated() -> None: # Older buckets were dropped, and everything before the oldest retained @@ -535,21 +618,17 @@ def _warn_truncated() -> None: interval, ) - try: - rows: list[dict] = await collect_pages( - partial( - client.bot_orchestration.get_controller_performance_history, - bot_name=instance_name, - interval=interval, - ), - extract_history_rows, - page_size=limit, - max_items=max_rows, - on_truncated=_warn_truncated, - ) - except Exception as e: - logger.debug("fetch_instance_history(%s) failed: %s", instance_name, e) - return [] + rows: list[dict] = await collect_pages( + partial( + client.bot_orchestration.get_controller_performance_history, + bot_name=instance_name, + interval=interval, + ), + extract_history_rows, + page_size=limit, + max_items=max_rows, + on_truncated=_warn_truncated, + ) # One cumulative series per controller. A repeated (controller, timestamp) # keeps the last value read, which is what the endpoint means by re-reporting @@ -645,6 +724,52 @@ def slice_history( return realized, volume, trades, fees +def slice_history_series( + histories: list[list[tuple[float, float, float, float, float]]], + start: float, + stamps: list[float], +) -> list[tuple[float, float, float, float, float]]: + """:func:`slice_history` evaluated at every instant of ascending ``stamps``. + + Identical output to ``[slice_history(histories, start, t) for t in stamps]`` + — same per-instance differencing in the same order — but computed in one + merge pass: the ``cum_at(start)`` baseline is constant across stamps so it + is taken once per instance, and each instance's cursor only ever advances + as the stamps increase (the same carry idiom + :func:`fetch_instance_history` uses internally). That turns the naive + O(stamps × rows) rescan into O(stamps + rows) per instance, which matters + on the per-tick and per-request paths that rebuild whole-session curves + from near-cap histories. + + Returns ``[(t, realized, volume, trades, fees), …]``, one row per stamp. + """ + bases = [_cum_at(h, start) for h in histories] + cursors = [-1] * len(histories) + carried: list[tuple[float, float, float, float]] = [(0.0, 0.0, 0.0, 0.0)] * len( + histories + ) + + out: list[tuple[float, float, float, float, float]] = [] + for t in stamps: + realized = volume = trades = fees = 0.0 + for i, h in enumerate(histories): + j = cursors[i] + while j + 1 < len(h) and h[j + 1][0] <= t: + j += 1 + if j != cursors[i]: + cursors[i] = j + row = h[j] + carried[i] = (row[1], row[2], row[3], row[4]) + r_e, v_e, t_e, f_e = carried[i] + r_s, v_s, t_s, f_s = bases[i] + realized += r_e - r_s + volume += v_e - v_s + trades += t_e - t_s + fees += f_e - f_s + out.append((t, realized, volume, trades, fees)) + return out + + # One shared fetch of every owned base's instance histories, at a resolution that # actually spans the ownership timeline. Both attribution callers go through this # so the dashboard's per-session slice and the live agent's own view of its PnL diff --git a/condor/fetchers/gateway_tokens.py b/condor/fetchers/gateway_tokens.py new file mode 100644 index 000000000..b274f9e4f --- /dev/null +++ b/condor/fetchers/gateway_tokens.py @@ -0,0 +1,222 @@ +"""Put a pool's tokens in Gateway's token list before anything tries to read them. + +Gateway resolves a *swap* by mint address on its own — Solana's ``getToken`` +falls back to reading the mint on chain, Token-2022 included — which is why a +pool discovered on GeckoTerminal can be traded at all without the user +maintaining anything by hand. It will not resolve a *balance* that way: +``fetchTokenAccounts`` skips every mint that is not in the network's token list, +so ``processSpecificTokens`` answers 0 even when the balance is asked for by +address. Everything that sizes an order off a balance — a sell, the percentage +presets, an LP amount, the portfolio row — therefore sees an empty wallet until +the token is listed. On EVM there is no on-chain fallback at all and the swap +itself fails with "token not found". + +So the token list is not a convenience: it is a precondition of trading a pool, +and one the user should never have to satisfy by hand. Condor satisfies it the +moment a pool workspace opens, and again when an executor is created, for the +callers (agents, Telegram, MCP) that never open one. +""" + +from __future__ import annotations + +import logging +from typing import Any, Iterable + +from condor.dex_candles import ADDRESS_RE + +logger = logging.getLogger(__name__) + +# What one ensure call is allowed to register. A pool has two tokens; anything +# beyond a handful is a caller passing a list it did not mean to. +MAX_TOKENS_PER_CALL = 8 + +# Addresses this process has already seen on the list, so opening the same pool +# twice costs nothing. Only *confirmed* listings are remembered — never a failure, +# which must stay retryable — so the memo can never mask a missing token. Bounded +# because a browsing session mints a fresh entry per pool visited. +_MAX_MEMO = 4000 +_listed: set[tuple[str, str]] = set() + + +def reset_listed_memo() -> None: + """Forget what is known to be listed. For tests, and for a Gateway swap.""" + _listed.clear() + + +def forget_listed(network_id: str, address: str) -> None: + """Drop one confirmed listing, after its token is deleted from the list. + + Without this, replacing a ticker's holder would leave the deleted address + memoized as listed — and every pool that uses it would trust a balance + Gateway can no longer read. + """ + _listed.discard(_memo_key(network_id, address)) + + +def _memo_key(network_id: str, address: str) -> tuple[str, str]: + return (network_id, address.lower()) + + +def _remember(network_id: str, address: str) -> None: + if len(_listed) >= _MAX_MEMO: + _listed.clear() + _listed.add(_memo_key(network_id, address)) + + +def token_addresses(*values: str | None) -> list[str]: + """The address-shaped values among ``values``, deduped, order preserved. + + Takes anything a caller happens to hold — a trading pair's two sides, a + pool's two mints — and keeps what can actually be registered. A ticker quote + (``USDC``, or an XRPL pair whose base is a symbol) is not an address and is + dropped rather than sent upstream to fail. + """ + seen: set[str] = set() + addresses: list[str] = [] + for value in values: + for part in str(value or "").split("-"): + candidate = part.strip() + if not candidate or not ADDRESS_RE.match(candidate): + continue + if candidate.lower() in seen: + continue + seen.add(candidate.lower()) + addresses.append(candidate) + return addresses + + +async def _network_tokens(client: Any, network_id: str) -> list[dict]: + """Gateway's whole token list for ``network_id``. + + Read whole rather than filtered because Gateway's ``search`` only looks at + symbol and name — a mint address matches nothing, and a ticker matches by + rules this module should not have to guess at. The list is a curated file + per network, so the whole of it is a small response. + """ + response = await client.gateway.get_network_tokens(network_id) + tokens = response.get("tokens", []) if isinstance(response, dict) else response + return [token for token in (tokens or []) if isinstance(token, dict)] + + +async def _is_listed(client: Any, network_id: str, address: str) -> bool: + """Whether Gateway's token list for ``network_id`` already holds ``address``. + + The whole list is read, not ``search=
``: Gateway matches ``search`` + against symbol and name only, so asking for a mint always answered zero + tokens. Every token then looked unlisted, the save that followed was refused + with "already exists" (a *symbol* check, which the listed token's own ticker + trips), and the re-read said unlisted again — so a correctly listed token + like Solana USDC reported ``symbol_taken`` forever, the pool banner asked the + user to add a token that was already there, and the conflict lookup could + name no holder but the token itself. + + A network's list is Gateway's curated file — tens of entries, not the whole + chain — so reading it whole is one small response, and confirmed hits are + memoized by the caller anyway. + """ + wanted = address.lower() + return any( + str((token or {}).get("address") or "").lower() == wanted + for token in await _network_tokens(client, network_id) + ) + + +async def find_symbol_holder( + client: Any, network_id: str, symbol: str, exclude: str | None = None +) -> dict[str, Any] | None: + """The token on ``network_id``'s list that holds ``symbol``, if any. + + This is the other half of a ``symbol_taken`` verdict: the verdict says the + ticker is in use, this says by *whom* — which is what the user needs to + decide whether the holder should be replaced. ``exclude`` skips the token + being registered itself, so a half-written state cannot name it as its own + conflict. + """ + wanted = symbol.lower() + excluded = (exclude or "").lower() + for entry in await _network_tokens(client, network_id): + if str(entry.get("symbol") or "").lower() != wanted: + continue + address = str(entry.get("address") or "") + if address.lower() == excluded: + continue + return { + "symbol": entry.get("symbol"), + "address": address, + "name": entry.get("name"), + } + return None + + +async def ensure_tokens_listed( + client: Any, network_id: str, addresses: Iterable[str] +) -> dict[str, str]: + """Register every address that Gateway does not already know, idempotently. + + Returns one verdict per address: + + - ``listed`` — already there, nothing was written. + - ``added`` — fetched from GeckoTerminal by Gateway and saved. + - ``symbol_taken`` — another token on the list already holds this one's + ticker, so Gateway refused to save it. Balances for it will keep reading 0, + and a *quote* in that state resolves to the other token's address, which is + a wrong-token trade rather than a failed one. Reported, never forced: the + fix is a human deciding which token owns the ticker. + - ``failed`` — the lookup or the save errored; retryable, so nothing is + remembered. + + Registration is sequential on purpose. Gateway's ``addToken`` is a + read-modify-write of one JSON file, so two concurrent saves for the same + network can drop one of the two tokens. + """ + verdicts: dict[str, str] = {} + for address in list(addresses)[:MAX_TOKENS_PER_CALL]: + if _memo_key(network_id, address) in _listed: + verdicts[address] = "listed" + continue + + try: + if await _is_listed(client, network_id, address): + _remember(network_id, address) + verdicts[address] = "listed" + continue + except Exception as e: + logger.warning("token lookup failed %s on %s: %s", address, network_id, e) + verdicts[address] = "failed" + continue + + try: + # Address in, token out: Gateway resolves symbol, name and decimals + # from GeckoTerminal itself, so Condor spends none of its own budget + # and stores no metadata it would then have to keep fresh. + await client.gateway.save_network_token( + network_id=network_id, token_address=address + ) + except Exception as e: + logger.warning("token save failed %s on %s: %s", address, network_id, e) + verdicts[address] = "failed" + continue + + # A save whose symbol collides with a token already on the list answers + # 200 with "already exists" and writes nothing, so the response cannot be + # trusted to mean the token is there. Re-reading the list can. + try: + saved = await _is_listed(client, network_id, address) + except Exception as e: + logger.warning("token re-check failed %s on %s: %s", address, network_id, e) + verdicts[address] = "failed" + continue + + if saved: + _remember(network_id, address) + verdicts[address] = "added" + else: + logger.warning( + "token %s on %s not saved: its symbol is held by another address; " + "balances for it will read 0", + address, + network_id, + ) + verdicts[address] = "symbol_taken" + + return verdicts diff --git a/condor/fetchers/market_data.py b/condor/fetchers/market_data.py index 48da5b69f..2954dd7b6 100644 --- a/condor/fetchers/market_data.py +++ b/condor/fetchers/market_data.py @@ -9,15 +9,27 @@ async def fetch_current_price( - client, connector_name: str = "", trading_pair: str = "", **_kw + client, + connector_name: str = "", + trading_pair: str = "", + strict: bool = False, + **_kw, ) -> Optional[float]: - """Fetch current price for a trading pair.""" + """Fetch current price for a trading pair. + + Args: + strict: Re-raise a failed request instead of degrading to None. The SDS + registration wants the raise so a transient failure keeps the last + good cached price instead of caching (and broadcasting) None. + """ try: prices = await client.market_data.get_prices( connector_name=connector_name, trading_pairs=trading_pair ) return prices.get("prices", {}).get(trading_pair) except Exception as e: + if strict: + raise logger.warning("Error fetching price for %s: %s", trading_pair, e) return None @@ -325,7 +337,7 @@ def _normalize_tickers( return tickers, latest_ts -async def fetch_ticker_pool(client, **_kw) -> Dict[str, Any]: +async def fetch_ticker_pool(client, strict: bool = False, **_kw) -> Dict[str, Any]: """Fetch the API's whole ticker pool in a single call. Without a `connectors` filter the endpoint is a plain read of the pool the API @@ -334,6 +346,12 @@ async def fetch_ticker_pool(client, **_kw) -> Dict[str, Any]: and serves both per-connector tickers and currency conversion from it, instead of hitting `/market-data/tickers` and `/market-data/rates` per request. + Args: + strict: Re-raise a failed request instead of degrading to the empty pool, + so the SDS keeps the last good pool and backs off. A 404 ("server has + no tickers endpoint") stays an empty pool even in strict mode — that + is a real answer, not a failure (mirrors `fetch_rates`). + Returns: { "connectors": {connector: {pair: {price, base_volume, quote_volume, usd_volume}}}, @@ -348,8 +366,10 @@ async def fetch_ticker_pool(client, **_kw) -> Dict[str, Any]: error_str = str(e) if "404" in error_str or "not found" in error_str.lower(): logger.debug("Server has no /market-data/tickers endpoint: %s", e) - else: - logger.warning("Error fetching ticker pool: %s", e) + return empty + if strict: + raise + logger.warning("Error fetching ticker pool: %s", e) return empty raw_by_connector = (result or {}).get("tickers") or {} @@ -379,13 +399,21 @@ async def fetch_ticker_pool(client, **_kw) -> Dict[str, Any]: return {"connectors": connectors, "prices": prices, "updated_at": updated_at} -async def fetch_tickers(client, connector_name: str = "", **_kw) -> Dict[str, Any]: +async def fetch_tickers( + client, connector_name: str = "", strict: bool = False, **_kw +) -> Dict[str, Any]: """Fetch 24h tickers (price + volumes) for a single connector. Fallback for connectors that aren't in the cached pool yet: naming the connector makes the API fetch it on demand and enroll it in its background refresh cycle, so subsequent pool reads include it. Prefer `fetch_ticker_pool` where possible. + Args: + strict: Re-raise a failed request instead of degrading to empty tickers, + so the SDS keeps the last good value and backs off. A 404 ("server + has no tickers endpoint") stays a real empty answer even in strict + mode (mirrors `fetch_ticker_pool`). + Returns: {"tickers": {pair: {price, base_volume, quote_volume, usd_volume}}, "updated_at": float|None} """ @@ -402,8 +430,10 @@ async def fetch_tickers(client, connector_name: str = "", **_kw) -> Dict[str, An connector_name, e, ) - else: - logger.warning("Error fetching tickers for %s: %s", connector_name, e) + return {"tickers": {}, "updated_at": None} + if strict: + raise + logger.warning("Error fetching tickers for %s: %s", connector_name, e) return {"tickers": {}, "updated_at": None} raw = ((result or {}).get("tickers") or {}).get(connector_name) or {} diff --git a/condor/frontmatter.py b/condor/frontmatter.py new file mode 100644 index 000000000..5e47ea34b --- /dev/null +++ b/condor/frontmatter.py @@ -0,0 +1,68 @@ +"""YAML-frontmatter round-trip and slug helpers — the one implementation. + +Every markdown-with-frontmatter file Condor persists (``AGENT.md``, +``strategy.md``, memory facts, skill playbooks) goes through this module, so +parse and render can never drift apart between callers. Consolidated from the +near-verbatim private copies in ``condor/agents/strategy.py`` and +``condor/memory/store.py`` (ARCH-202), following the same pattern that +collapsed the atomic-write copies into :mod:`condor.fsutil` (ARCH-148). + +The rendered format is byte-identical to what both former copies produced +(``yaml.dump`` with ``default_flow_style=False``, ``allow_unicode=True``, +``sort_keys=False``), so existing on-disk files round-trip unchanged. +""" + +from __future__ import annotations + +import logging +import re + +import yaml + +log = logging.getLogger(__name__) + + +def slugify(name: str, fallback: str = "unnamed") -> str: + """Convert a name to a filesystem-safe slug. + + Example: "RIVER Scalper v2" -> "river_scalper_v2" + + ``fallback`` is returned when nothing sluggable remains (empty or + all-punctuation names); callers pass their historical word so existing + on-disk slugs are preserved (agents/strategies use the default + ``"unnamed"``, the memory/skill stores use ``"memory"``). + """ + s = name.lower().strip() + s = re.sub(r"[^\w\s-]", "", s) + s = re.sub(r"[\s-]+", "_", s) + return s.strip("_") or fallback + + +def parse_frontmatter(text: str) -> tuple[dict, str]: + """Parse YAML frontmatter and markdown body from a file.""" + text = text.strip() + if not text.startswith("---"): + return {}, text + + end = text.find("---", 3) + if end == -1: + return {}, text + + frontmatter_str = text[3:end].strip() + body = text[end + 3 :].strip() + + try: + meta = yaml.safe_load(frontmatter_str) or {} + except yaml.YAMLError: + log.warning("Failed to parse YAML frontmatter") + meta = {} + + return meta, body + + +def render_frontmatter(meta: dict, body: str) -> str: + """Render YAML frontmatter + markdown body.""" + frontmatter = yaml.dump( + meta, default_flow_style=False, allow_unicode=True, sort_keys=False + ).strip() + return f"---\n{frontmatter}\n---\n\n{body}\n" diff --git a/condor/llm/__init__.py b/condor/llm/__init__.py new file mode 100644 index 000000000..76d71e0ea --- /dev/null +++ b/condor/llm/__init__.py @@ -0,0 +1,14 @@ +"""Platform-neutral LLM/model-catalog foundations (ARCH-190). + +Everything here answers "which models can Condor run, and are they ready?" for +every surface — the Telegram bot, the web dashboard, the setup wizard and the +MCP tools. Nothing in this package may import from ``handlers/`` or +``condor.web``: the layering is handlers -> condor, never the reverse. + +Modules: +- :mod:`condor.llm.options` — the agent/model catalog (``AGENT_OPTIONS``) and + the resolved default (``DEFAULT_AGENT``). +- :mod:`condor.llm.readiness` — "can this machine actually run that model?". +- :mod:`condor.llm.openrouter_models` — the OpenRouter catalog fetch. +- :mod:`condor.llm.custom_models` — OpenAI-compatible custom endpoints. +""" diff --git a/condor/llm/custom_models.py b/condor/llm/custom_models.py new file mode 100644 index 000000000..b8f212d35 --- /dev/null +++ b/condor/llm/custom_models.py @@ -0,0 +1,344 @@ +"""Custom OpenAI-compatible provider model fetcher. + +Hits GET {base_url}/models with an optional Bearer key — the standard model +list endpoint every OpenAI-compatible API exposes (Venice AI, Together, +Fireworks, vLLM, TGI, ...). A successful fetch doubles as validation of both +the URL and the API key during the custom-endpoint setup flow. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import ipaddress +import logging +import os +import re +import socket +from urllib.parse import urlparse + +import aiohttp + +log = logging.getLogger(__name__) + +# Per-candidate budget. Candidates are probed concurrently, so this is also +# roughly the worst case for the whole fetch — the user is staring at a +# "validating..." message the entire time. +FETCH_TIMEOUT_SECONDS = 8 + +# Hosts that hand out cloud instance credentials to anything that can issue an +# HTTP GET. Always refused: no legitimate model endpoint lives here. +BLOCKED_HOSTNAMES = frozenset( + { + "metadata.google.internal", + "metadata.goog", + "instance-data", + } +) + +# Opt-in for multi-tenant deployments: also refuse loopback and RFC1918 +# targets. Off by default because pointing at a local Ollama/vLLM/LM Studio +# server is the single most common use of this feature. +_BLOCK_PRIVATE_ENV = "CUSTOM_LLM_BLOCK_PRIVATE_URLS" + +# /models entries whose declared type isn't one of these are not chat models. +# Providers disagree on the field name (`type` on Venice/Together, `object` on +# stock OpenAI) and on the vocabulary, so accept every spelling of "text in, +# text out" we've seen and treat other declared values as non-chat. +CHAT_MODEL_TYPES = frozenset({"text", "chat", "language", "llm"}) + +# Values that technically occupy the type field but say nothing about +# modality — stock OpenAI stamps object="model" on embeddings, Whisper and +# DALL·E alike. Treat these as "not declared" and fall through to the id. +UNINFORMATIVE_MODEL_TYPES = frozenset({"model", "list", "object"}) + +# Fallback for providers that return no type at all: drop ids that are +# unmistakably a non-chat modality. +_NON_CHAT_MARKERS = ( + "embed", + "rerank", + "whisper", + "tts", + "text-to-speech", + "speech", + "voice", + "moderation", + "guard", + "upscal", + "stable-diffusion", + "flux", + "dall-e", + "sdxl", + "image", + "vision-encoder", + "clip", + "ocr", +) + + +# urlparse is happy to call "not a url at all" a netloc, so the hostname needs +# its own shape check: dotted labels, or an IPv6 literal in brackets. +_HOSTNAME_RE = re.compile( + r"^(?:[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*\.?|\[[0-9A-Fa-f:.]+\])$" +) + + +class CustomProviderError(Exception): + """Raised when the endpoint can't be validated. Message is user-facing.""" + + +def _block_private() -> bool: + return os.environ.get(_BLOCK_PRIVATE_ENV, "").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) + + +def _reject_if_blocked(host: str) -> None: + """Raise ValueError when `host` points somewhere we refuse to fetch from. + + Link-local addresses (the 169.254.169.254 cloud metadata range) are always + refused. Loopback and private ranges are allowed by default so local model + servers work, and can be refused too by setting + CUSTOM_LLM_BLOCK_PRIVATE_URLS=true on shared/multi-tenant deployments. + + Only IP literals are checked unless the strict flag is set — resolving + every hostname would add latency and would still be defeatable by DNS + rebinding, so this is a guard against casual probing, not a security + boundary. + """ + lowered = host.lower().rstrip(".") + if lowered in BLOCKED_HOSTNAMES: + raise ValueError(f"'{host}' is not an allowed endpoint host.") + + strict = _block_private() + addresses: list[str] = [] + try: + ipaddress.ip_address(lowered.strip("[]")) + addresses = [lowered.strip("[]")] + except ValueError: + if strict: + try: + addresses = [info[4][0] for info in socket.getaddrinfo(lowered, None)] + except OSError: + # Unresolvable — let the actual request produce the error + return + + for raw in addresses: + try: + ip = ipaddress.ip_address(raw) + except ValueError: + continue + if ip.is_link_local: + raise ValueError( + f"'{host}' resolves to a link-local address, which is blocked " + "(cloud instance metadata lives there)." + ) + if strict and (ip.is_loopback or ip.is_private or ip.is_reserved): + raise ValueError( + f"'{host}' resolves to a private address and " + f"{_BLOCK_PRIVATE_ENV} is enabled." + ) + + +def normalize_base_url(raw: str) -> str: + """Coerce user input into a usable OpenAI-compatible base URL. + + Adds https:// when no scheme is given and strips trailing slashes and + endpoint suffixes users commonly paste (/chat/completions, /models). + Raises ValueError for input that can't be a URL or that targets a + blocked host. + """ + url = raw.strip() + if not url: + raise ValueError("URL is empty.") + if "://" not in url: + url = f"https://{url}" + url = url.rstrip("/") + + try: + parsed = urlparse(url) + hostname = parsed.hostname + except ValueError: + raise ValueError(f"'{raw.strip()}' is not a valid http(s) URL.") from None + if ( + parsed.scheme not in ("http", "https") + or not parsed.netloc + or not hostname + or not _HOSTNAME_RE.match(hostname) + ): + raise ValueError(f"'{raw.strip()}' is not a valid http(s) URL.") + _reject_if_blocked(hostname) + + for suffix in ("/chat/completions", "/completions", "/models"): + if url.endswith(suffix): + url = url[: -len(suffix)] + break + return url.rstrip("/") + + +def _is_chat_model(entry: dict) -> bool: + """Best-effort guess at whether a /models entry can serve chat completions. + + Providers return embeddings, rerankers, TTS and image models from the same + endpoint. Picking one of those succeeds here and then fails at prompt time + with an opaque upstream error, so filter them out while we still have the + metadata to do it. + """ + declared = entry.get("type") or entry.get("object") + if isinstance(declared, str) and declared.strip(): + normalized = declared.strip().lower() + if normalized not in UNINFORMATIVE_MODEL_TYPES: + return normalized in CHAT_MODEL_TYPES + + capabilities = entry.get("capabilities") + if isinstance(capabilities, dict): + for flag in ("chat", "chat_completions", "supportsChatCompletions"): + if flag in capabilities: + return bool(capabilities[flag]) + + model_id = entry.get("id", "").lower() + return not any(marker in model_id for marker in _NON_CHAT_MARKERS) + + +def _extract_model_ids(payload: object, chat_only: bool = True) -> list[str]: + """Pull model ids from an OpenAI-format /models response. + + Expects {"data": [{"id": "..."}, ...]}. Returns sorted, deduped ids; + empty list when the payload doesn't match the expected shape. With + ``chat_only`` (the default) non-chat modalities are dropped. + """ + if not isinstance(payload, dict): + return [] + data = payload.get("data") + if not isinstance(data, list): + return [] + ids = { + entry["id"].strip() + for entry in data + if isinstance(entry, dict) + and isinstance(entry.get("id"), str) + and entry["id"].strip() + and (not chat_only or _is_chat_model(entry)) + } + return sorted(ids, key=str.lower) + + +async def _probe( + session: aiohttp.ClientSession, candidate: str, headers: dict +) -> tuple[list[str], CustomProviderError | None]: + """Fetch one candidate's model list. Returns (models, error).""" + models_url = f"{candidate}/models" + try: + async with session.get(models_url, headers=headers) as resp: + if resp.status in (401, 403): + return [], CustomProviderError( + f"the provider at {candidate} rejected the API key " + f"(HTTP {resp.status}). Check the key and try again." + ) + if resp.status == 404: + return [], CustomProviderError( + f"no /models endpoint at {candidate} (HTTP 404). " + "Check the base URL — it usually ends in /v1." + ) + if resp.status != 200: + return [], CustomProviderError( + f"the provider at {candidate} returned HTTP {resp.status}." + ) + payload = await resp.json(content_type=None) + except asyncio.CancelledError: + raise + except Exception as e: + return [], CustomProviderError( + f"could not reach {models_url} ({type(e).__name__}). " + "Check the URL and that the server is up." + ) + + model_ids = _extract_model_ids(payload) + if not model_ids: + # An endpoint that answered but listed only embeddings/image models is + # a different failure from one that isn't OpenAI-shaped at all. + if _extract_model_ids(payload, chat_only=False): + return [], CustomProviderError( + f"{models_url} returned models, but none of them are chat " + "models (only embeddings/image/audio)." + ) + return [], CustomProviderError( + f"{models_url} responded but did not return an " + "OpenAI-compatible model list." + ) + return model_ids, None + + +async def fetch_models(base_url: str, api_key: str = "") -> tuple[str, list[str]]: + """Validate an OpenAI-compatible endpoint and return its chat model list. + + Probes {base_url}/models and {base_url}/v1/models concurrently — the second + covers users who pasted the host root (e.g. https://api.venice.ai instead + of https://api.venice.ai/api/v1) — and prefers the first candidate that + works. Returns (resolved_base_url, model_ids). + + Raises CustomProviderError with a user-facing message on any failure — + unreachable host, rejected key, or a non-OpenAI-shaped response. + """ + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + candidates = [base_url] + if not base_url.endswith("/v1"): + candidates.append(f"{base_url}/v1") + + timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(timeout=timeout) as session: + results = await asyncio.gather( + *(_probe(session, candidate, headers) for candidate in candidates) + ) + + # Candidate order is preference order: the URL the user actually typed wins + for candidate, (model_ids, _) in zip(candidates, results): + if model_ids: + log.info( + "Custom provider validated: %s (%d chat models)", + candidate, + len(model_ids), + ) + return candidate, model_ids + + for _, error in results: + if error: + raise error + raise CustomProviderError("no reachable endpoint found.") + + +def model_token(model_id: str) -> str: + """Short stable token for a model id, for use in callback_data. + + Telegram caps callback_data at 64 bytes and model ids are unbounded. A + content-derived token also survives the model list being refetched between + render and tap, which a bare list index does not. + """ + return hashlib.blake2s(model_id.encode("utf-8"), digest_size=4).hexdigest() + + +def find_by_token(model_ids: list[str], token: str) -> str | None: + """Resolve a token produced by :func:`model_token` back to its model id.""" + for model_id in model_ids: + if model_token(model_id) == token: + return model_id + return None + + +def format_model_label(model_id: str, limit: int = 38) -> str: + """Short label for inline keyboard buttons. + + Truncates through the middle: for ids like + ``meta-llama/Llama-3.3-70B-Instruct-Turbo-Free`` the tail is what + distinguishes one variant from another, so cutting it off is exactly the + wrong end to lose. + """ + if len(model_id) <= limit: + return model_id + keep = limit - 1 # room for the ellipsis + head = (keep + 1) // 2 + tail = keep - head + return f"{model_id[:head]}…{model_id[-tail:]}" diff --git a/condor/llm/openrouter_models.py b/condor/llm/openrouter_models.py new file mode 100644 index 000000000..8a7b17b58 --- /dev/null +++ b/condor/llm/openrouter_models.py @@ -0,0 +1,119 @@ +"""OpenRouter model catalog fetcher. + +Hits GET https://openrouter.ai/api/v1/models (public, unauthenticated) and +filters to entries that advertise `tools` in their supported_parameters. +Condor's whole architecture depends on tool-calling, so models without it +are hidden from the picker. + +Results are cached for an hour so paginating the picker doesn't refetch. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass + +import aiohttp + +log = logging.getLogger(__name__) + +OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" +CACHE_TTL_SECONDS = 3600 + + +@dataclass(frozen=True) +class OpenRouterModel: + slug: str # e.g. "anthropic/claude-sonnet-4-5" + name: str # human-friendly name from the API + context_length: int # tokens + prompt_price: float # USD per 1M input tokens, 0 if free + completion_price: float + + +_cache: tuple[float, list[OpenRouterModel]] | None = None + + +def _parse_price(value: object) -> float: + """OpenRouter returns prices as strings in USD per token. Convert to per-1M.""" + try: + return float(value) * 1_000_000 # type: ignore[arg-type] + except (TypeError, ValueError): + return 0.0 + + +def _model_supports_tools(entry: dict) -> bool: + params = entry.get("supported_parameters") or [] + return "tools" in params + + +async def fetch_models(force_refresh: bool = False) -> list[OpenRouterModel]: + """Return the list of OpenRouter models that support tool-calling. + + Sorted by provider then model id for predictable pagination. + """ + global _cache + + if not force_refresh and _cache is not None: + cached_at, models = _cache + if time.monotonic() - cached_at < CACHE_TTL_SECONDS: + return models + + timeout = aiohttp.ClientTimeout(total=10) + try: + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(OPENROUTER_MODELS_URL) as resp: + resp.raise_for_status() + payload = await resp.json() + except Exception as e: + log.warning("Failed to fetch OpenRouter models: %s", e) + if _cache is not None: + return _cache[1] # serve stale cache on failure + return [] + + raw = payload.get("data") or [] + models: list[OpenRouterModel] = [] + for entry in raw: + if not isinstance(entry, dict): + continue + slug = entry.get("id") + if not isinstance(slug, str) or not slug or slug.startswith("~"): + continue # skip canonical aliases like "~anthropic/claude-haiku-latest" + if not _model_supports_tools(entry): + continue + pricing = entry.get("pricing") or {} + models.append( + OpenRouterModel( + slug=slug, + name=str(entry.get("name") or slug), + context_length=int(entry.get("context_length") or 0), + prompt_price=_parse_price(pricing.get("prompt")), + completion_price=_parse_price(pricing.get("completion")), + ) + ) + + models.sort(key=lambda m: m.slug.lower()) + _cache = (time.monotonic(), models) + log.info("Fetched %d OpenRouter models with tool support", len(models)) + return models + + +def format_button_label(model: OpenRouterModel) -> str: + """Short label for inline keyboard buttons. Telegram max ~30 chars looks good.""" + label = model.name or model.slug + if len(label) > 38: + label = label[:35] + "..." + return label + + +def find_model_by_slug( + models: list[OpenRouterModel], slug: str +) -> OpenRouterModel | None: + """Case-insensitive exact-match lookup by slug.""" + target = slug.strip().lower() + if not target: + return None + for m in models: + if m.slug.lower() == target: + return m + return None diff --git a/condor/llm/options.py b/condor/llm/options.py new file mode 100644 index 000000000..527b437ae --- /dev/null +++ b/condor/llm/options.py @@ -0,0 +1,83 @@ +"""The agent/model catalog and the resolved default (ARCH-190). + +One catalog (``AGENT_OPTIONS``) for every surface that offers a model — the +Telegram menu, the web dashboard's dropdown, the setup wizard. Moved here from +``handlers/agents/_shared.py``, which re-exports these names for its Telegram +callers. +""" + +from __future__ import annotations + +import os +from typing import Any + +from condor.memory.paths import CHAT_SLUG + + +def _chat_agent(): + """The default agent's record — Condor's ``agents/condor/AGENT.md``. + + There is one loader and one frontmatter schema (FEAT-033): the chat is read + by ``AgentStore`` exactly like a specialist is. Returns ``None`` only if the + file is missing or unreadable, which callers treat as "no instructions" + rather than as a startup failure. + """ + from condor.agents.agent import AgentStore + + return AgentStore().get(CHAT_SLUG) + + +AGENT_OPTIONS: dict[str, dict[str, Any]] = { + "claude-code": {"label": "Claude Code"}, + "claude-acp:opus": {"label": "Claude (ACP) — Opus"}, + "claude-acp:sonnet": {"label": "Claude (ACP) — Sonnet"}, + "gemini": {"label": "Gemini CLI"}, + "copilot": {"label": "GitHub Copilot CLI"}, + "codex": {"label": "ChatGPT Codex"}, + "ollama:": {"label": "Ollama — Default Model"}, + "lmstudio:": {"label": "LM Studio — Default Model"}, + # Sentinels — these open a picker instead of being a selectable model, so + # they are NOT valid agent keys. `picker` marks them for surfaces that + # render AGENT_OPTIONS as a flat list of choices (the web dashboard's model + # dropdown), which would otherwise offer a key that fails at session start. + # + # OpenRouter: stored agent_llm becomes "openrouter:". + "openrouter:": {"label": "OpenRouter — Pick Model", "picker": True}, + # Custom endpoints: the user's saved OpenAI-compatible endpoints live in + # preferences (condor/preferences.py, shared with the web dashboard) and + # the stored agent_llm becomes "custom@:". + "custom:": {"label": "Custom — OpenAI-compatible API", "picker": True}, +} + + +# What a fresh install should pick unless it has a reason not to: the ACP bridge +# is the best-supported path, and Sonnet is the balance of capability and cost +# most users want. Surfaces that rank the catalog (the `make pick-model` wizard) +# float this key to the top and mark it; it is deliberately NOT part of the +# `label`, which is reused as a plain display name all over the bot. +RECOMMENDED_AGENT = "claude-acp:sonnet" + + +def selectable_agent_options() -> dict[str, dict[str, Any]]: + """AGENT_OPTIONS minus the picker sentinels — every key here is startable.""" + return {k: v for k, v in AGENT_OPTIONS.items() if not v.get("picker")} + + +def _default_agent() -> str: + """Resolve the fallback agent_key for users who haven't picked a model. + + Precedence: ``CONDOR_DEFAULT_AGENT`` env > Condor's ``AGENT.md`` frontmatter + ``agent_key`` > ``"claude-code"``. A user's own /agent → Change LLM choice + (``agent_llm``) still overrides this at runtime; this is only the default. + Examples for the frontmatter / env: ``claude-code``, ``claude-acp:opus``, + ``ollama:qwen3:32b``. + """ + agent = _chat_agent() + return ( + os.environ.get("CONDOR_DEFAULT_AGENT") + or (agent.agent_key if agent else "") + or "claude-code" + ) + + +DEFAULT_AGENT = _default_agent() diff --git a/condor/llm/readiness.py b/condor/llm/readiness.py new file mode 100644 index 000000000..340177322 --- /dev/null +++ b/condor/llm/readiness.py @@ -0,0 +1,298 @@ +"""One answer to "can Condor actually run this model right now?". + +Every surface that offers a model — the ``get_available_models`` MCP tool behind +the agent_builder skill, and the setup wizard (``condor/setup_llm.py``) — asks +this module, so they can never disagree about what a machine can run. The probes +source their commands from :data:`condor.acp.ACP_COMMANDS` and their base URLs +from :data:`condor.acp.pydantic_ai_client.DEFAULT_BASE_URLS`, so adding a +provider there lights it up here for free. + +Three states, never two: a CLI bridge that is installed but whose login cannot be +proven is ``UNVERIFIED``, not ready and not missing. The login markers below are +undocumented paths that upstream CLIs may move, so they only ever downgrade a row +from *ready* to *unverified* — never to *missing*, and never enough to refuse a +pick. + +Read-only and side-effect-free. Never returns a key value, and an unreachable +local server is a normal state (the user simply isn't running it), reported as +``MISSING`` with the command that starts it, never raised. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable + +import httpx + +from condor.acp import ACP_COMMANDS +from condor.acp.pydantic_ai_client import DEFAULT_BASE_URLS + +READY, UNVERIFIED, MISSING = "ready", "unverified", "missing" + +# Cloud providers Condor can address as ":", mapped to the env +# var holding each key. Presence of the key = the provider is usable. Sourced +# here (not hardcoded per surface) so the set stays in one place. +CLOUD_KEY_ENVS = { + "openrouter": "OPENROUTER_API_KEY", + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "groq": "GROQ_API_KEY", + "google": "GOOGLE_API_KEY", +} + +# Local, keyless OpenAI-compatible servers. Base URLs come from the resolver's +# own defaults (DEFAULT_BASE_URLS) so they can never drift from what actually +# runs the model at request time. +LOCAL_PREFIXES = ("ollama", "lmstudio") + +# What to run once, interactively, to log a bridge in. Not run from here: the +# wizard reports and instructs, it never drives someone else's OAuth flow. +_LOGIN_COMMANDS = { + "claude-code": "claude", + "claude-acp": "claude", + "gemini": "npx @google/gemini-cli", + "copilot": "npx @github/copilot", + "codex": "codex", +} + +# Bridges launched as a bare command still come from npm; name the package so a +# missing one is actionable instead of just absent. +_BARE_INSTALL = { + "claude-agent-acp": "npm install -g @agentclientprotocol/claude-agent-acp", +} + +_START_COMMANDS = { + "ollama": "start it with `ollama serve`", + "lmstudio": "start LM Studio and enable its local server", +} + + +@dataclass(frozen=True) +class Readiness: + """What a menu row needs: a state, a sentence, and any models behind it.""" + + state: str # READY | UNVERIFIED | MISSING + detail: str # human sentence; when not READY it names the fix + models: list[str] = field(default_factory=list) # local server models; [] elsewhere + + @property + def usable(self) -> bool: + """False only for MISSING — an unproven login must never block a pick.""" + return self.state != MISSING + + +def npx_packages_installed() -> set[str]: + """npm packages resolvable WITHOUT a network install: local, global, npx cache. + + Mirrors how npx resolves a package before falling back to downloading it. + ``shutil.which("npx")`` says nothing about the package — npx exists whenever + Node does — so probing the launcher would mark every ``npx``-run bridge + available on any machine with Node. + """ + roots = [Path.cwd() / "node_modules"] + npm = shutil.which("npm") + if npm: + try: + proc = subprocess.run( + [npm, "root", "-g"], capture_output=True, text=True, timeout=10 + ) + if proc.returncode == 0 and proc.stdout.strip(): + roots.append(Path(proc.stdout.strip())) + except (OSError, subprocess.SubprocessError): + pass # npm unusable = nothing installed via it; local + cache still count + roots.extend((Path.home() / ".npm" / "_npx").glob("*/node_modules")) + + installed: set[str] = set() + for root in roots: + if not root.is_dir(): + continue + for entry in root.iterdir(): + if entry.name.startswith("@"): + installed.update(f"{entry.name}/{sub.name}" for sub in entry.iterdir()) + else: + installed.add(entry.name) + return installed + + +def _keychain_has(service: str) -> bool | None: + """macOS keychain lookup: True found, False absent, None cannot tell.""" + security = shutil.which("security") + if not security: + return None + try: + proc = subprocess.run( + [security, "find-generic-password", "-s", service], + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return None + return proc.returncode == 0 + + +def acp_login_state(base: str) -> bool | None: + """Has this bridge's interactive login already happened? + + ``True`` a credential exists, ``False`` none was found where that CLI keeps + one, ``None`` this bridge has no marker we can read (GitHub Copilot) or the + machine could not be asked. Heuristic by construction: a credentials file + proves a login happened, not that the token is still live, and its absence + proves nothing on a machine using another auth path — which is why callers + treat False and None alike as *unverified*, never as *not installed*. + """ + home = Path.home() + if base in ("claude-code", "claude-acp"): + if (home / ".claude" / ".credentials.json").exists(): + return True + if sys.platform == "darwin": # Claude Code stores it in the login keychain + return _keychain_has("Claude Code-credentials") + return False + if base == "gemini": + if (home / ".gemini" / "oauth_creds.json").exists(): + return True + return bool(os.environ.get("GEMINI_API_KEY")) or False + if base == "codex": + return (home / ".codex" / "auth.json").exists() + return None # copilot and anything new: no documented on-disk marker + + +def install_command(cmd: str) -> str: + """How to install the CLI behind an ACP command line.""" + parts = cmd.split() + if len(parts) > 1 and parts[0] == "npx": + return f"npm install -g {parts[1]}" + if parts and parts[0] in _BARE_INSTALL: + return _BARE_INSTALL[parts[0]] + return f"install `{cmd.split()[0] if parts else cmd}` and put it on PATH" + + +def acp_bridges() -> list[dict]: + """ACP CLI bridges: whether each is installed, and whether it is logged in. + + A bridge launched as ``npx …`` is available only if the PACKAGE is + already installed; a bare command is available if it is on PATH. ``available`` + means "launchable" — a bridge additionally needs its own interactive login, + reported separately as ``logged_in`` (``None`` = no marker to read). + """ + npx_packages = npx_packages_installed() + out: list[dict] = [] + for base, cmd in ACP_COMMANDS.items(): + parts = cmd.split() + if not parts: + available = False + elif parts[0] == "npx": + available = parts[1] in npx_packages if len(parts) > 1 else False + else: + available = shutil.which(parts[0]) is not None + out.append( + { + "agent_key": base, + "command": cmd, + "available": available, + "logged_in": acp_login_state(base) if available else None, + } + ) + return out + + +async def local_servers() -> dict: + """Probe local OpenAI-compatible servers and list their loaded models. + + Unreachable servers are reported (``reachable: false``), not raised — the + user just isn't running that server. + """ + result: dict = {} + timeout = httpx.Timeout(connect=1.5, read=3.0, write=1.5, pool=1.5) + async with httpx.AsyncClient(timeout=timeout) as client: + for prefix in LOCAL_PREFIXES: + base = DEFAULT_BASE_URLS.get(prefix) + entry = {"base_url": base, "reachable": False, "models": []} + if base: + try: + resp = await client.get(f"{base.rstrip('/')}/models") + if resp.status_code == 200: + data = resp.json().get("data", []) + entry["reachable"] = True + entry["models"] = sorted( + m["id"] for m in data if isinstance(m, dict) and m.get("id") + ) + except Exception: + pass # not running = normal, leave reachable=False + result[prefix] = entry + return result + + +def _bridge_readiness(entry: dict) -> Readiness: + base, cmd = entry["agent_key"], entry["command"] + if not entry["available"]: + return Readiness(MISSING, f"not installed — {install_command(cmd)}") + login = _LOGIN_COMMANDS.get(base, cmd) + if entry.get("logged_in") is True: + return Readiness(READY, "installed and logged in") + if entry.get("logged_in") is False: + return Readiness(UNVERIFIED, f"installed; run `{login}` once and log in") + return Readiness( + UNVERIFIED, + f"installed; login could not be verified — if your first chat " + f"fails, run `{login}` once and log in", + ) + + +def _local_readiness(prefix: str, entry: dict) -> Readiness: + if not entry["reachable"]: + hint = _START_COMMANDS.get(prefix, "start the server") + return Readiness(MISSING, f"not reachable at {entry['base_url']} — {hint}") + models = list(entry["models"]) + return Readiness(READY, f"{len(models)} model(s) available", models) + + +def _key_readiness(base: str, env: dict[str, str]) -> Readiness: + var = CLOUD_KEY_ENVS[base] + if env.get(var) or os.environ.get(var): + return Readiness(READY, f"{var} is set") + if base == "openrouter": + return Readiness(MISSING, "needs an API key (you can enter one here)") + return Readiness(MISSING, f"needs {var} in .env") + + +async def probe_all( + bases: Iterable[str], env: dict[str, str] | None = None +) -> dict[str, Readiness]: + """Readiness for several provider bases, probing each source exactly once. + + ``env`` is passed in rather than read from the process so the setup wizard + can probe against a ``.env`` the bot has not loaded yet; ``os.environ`` is + still consulted as a fallback. + """ + env = env or {} + wanted = list(dict.fromkeys(bases)) + bridges: dict[str, dict] = {} + if any(b in ACP_COMMANDS for b in wanted): + bridges = {e["agent_key"]: e for e in acp_bridges()} + servers: dict = {} + if any(b in LOCAL_PREFIXES for b in wanted): + servers = await local_servers() + + out: dict[str, Readiness] = {} + for base in wanted: + if base in bridges: + out[base] = _bridge_readiness(bridges[base]) + elif base in servers: + out[base] = _local_readiness(base, servers[base]) + elif base in CLOUD_KEY_ENVS: + out[base] = _key_readiness(base, env) + else: + out[base] = Readiness(UNVERIFIED, "cannot be checked from here") + return out + + +async def probe(base: str, env: dict[str, str] | None = None) -> Readiness: + """Readiness for a single provider base (see :func:`probe_all`).""" + return (await probe_all([base], env))[base] diff --git a/condor/memory/skills.py b/condor/memory/skills.py index be0a7817d..cf79ec886 100644 --- a/condor/memory/skills.py +++ b/condor/memory/skills.py @@ -61,8 +61,10 @@ import shutil from pathlib import Path +from condor.frontmatter import parse_frontmatter, render_frontmatter + from .paths import builtin_skills_root, shared_skills_root -from .store import _atomic_write, _parse_frontmatter, _render, _slugify, _utcnow +from .store import _atomic_write, _slugify, _utcnow def _routine_exists(name: str, agent_slug: str | None = None) -> bool: @@ -197,7 +199,7 @@ def create( # Preserve the original created date on overwrite. created = _utcnow() if path.exists(): - existing_meta, _ = _parse_frontmatter(path.read_text()) + existing_meta, _ = parse_frontmatter(path.read_text()) created = existing_meta.get("created", created) meta = { @@ -211,7 +213,7 @@ def create( if ref: meta["references_routine"] = ref - _atomic_write(path, _render(meta, body.strip())) + _atomic_write(path, render_frontmatter(meta, body.strip())) result = { "saved": True, "name": slug, @@ -247,7 +249,7 @@ def edit(self, name: str, **fields) -> dict: if not path.exists(): return {"error": f"Skill '{name}' not found"} - meta, body = _parse_frontmatter(path.read_text()) + meta, body = parse_frontmatter(path.read_text()) if fields.get("description"): meta["description"] = fields["description"].strip().replace("\n", " ") if fields.get("when_to_use"): @@ -264,7 +266,7 @@ def edit(self, name: str, **fields) -> dict: # `_target_dir` above already moved the folder if it changed. meta.pop("shared", None) - _atomic_write(path, _render(meta, body)) + _atomic_write(path, render_frontmatter(meta, body)) return self.read(slug) or {"saved": True, "name": slug} def delete(self, name: str) -> bool | dict: @@ -310,7 +312,7 @@ def read(self, name: str) -> dict | None: if skill_dir is None: return None path = skill_dir / "SKILL.md" - meta, body = _parse_frontmatter(path.read_text()) + meta, body = parse_frontmatter(path.read_text()) ref = meta.get("references_routine") result = { "name": meta.get("name", slug), @@ -496,7 +498,7 @@ def _iter_skills(self): if slug in seen: continue try: - meta, body = _parse_frontmatter(f.read_text()) + meta, body = parse_frontmatter(f.read_text()) except Exception: continue seen.add(slug) diff --git a/condor/memory/store.py b/condor/memory/store.py index 0284824eb..e820482a6 100644 --- a/condor/memory/store.py +++ b/condor/memory/store.py @@ -22,12 +22,10 @@ from __future__ import annotations import json -import re from datetime import datetime, timezone from pathlib import Path -import yaml - +from condor.frontmatter import parse_frontmatter, render_frontmatter, slugify from condor.fsutil import atomic_write_text from .paths import store_root @@ -41,40 +39,13 @@ def _slugify(name: str) -> str: - """Convert a memory name to a filesystem-safe slug.""" - s = name.lower().strip() - s = re.sub(r"[^\w\s-]", "", s) - s = re.sub(r"[\s-]+", "_", s) - return s.strip("_") or "memory" - - -def _parse_frontmatter(text: str) -> tuple[dict, str]: - """Parse YAML frontmatter and markdown body (mirrors strategy.py).""" - text = text.strip() - if not text.startswith("---"): - return {}, text - - end = text.find("---", 3) - if end == -1: - return {}, text - - frontmatter_str = text[3:end].strip() - body = text[end + 3 :].strip() + """Slug with this store's historical ``"memory"`` fallback. - try: - meta = yaml.safe_load(frontmatter_str) or {} - except yaml.YAMLError: - meta = {} - - return meta, body - - -def _render(meta: dict, body: str) -> str: - """Render YAML frontmatter + markdown body.""" - frontmatter = yaml.dump( - meta, default_flow_style=False, allow_unicode=True, sort_keys=False - ).strip() - return f"---\n{frontmatter}\n---\n\n{body}\n" + Thin binding over :func:`condor.frontmatter.slugify` (ARCH-202) so + memories and skills alike keep resolving empty/all-punctuation names to + the same on-disk slug they always have. + """ + return slugify(name, fallback="memory") def _utcnow() -> str: @@ -187,7 +158,7 @@ def write( existed = path.exists() created = _utcnow() if existed: - existing_meta, _ = _parse_frontmatter(path.read_text()) + existing_meta, _ = parse_frontmatter(path.read_text()) created = existing_meta.get("created", created) meta = { @@ -197,7 +168,7 @@ def write( "created": created, "source": source, } - _atomic_write(path, _render(meta, content.strip())) + _atomic_write(path, render_frontmatter(meta, content.strip())) self._reindex() action = "update" if existed else "create" self._append_audit(action, f"memory:{slug}", meta["description"], source) @@ -213,7 +184,7 @@ def read(self, name: str) -> str | None: path = self.memories_dir / f"{_slugify(name)}.md" if not path.exists(): return None - _, body = _parse_frontmatter(path.read_text()) + _, body = parse_frontmatter(path.read_text()) return body def search(self, query: str, limit: int = 10) -> list[dict]: @@ -261,7 +232,7 @@ def delete(self, name: str, source: str = "user") -> bool: path = self.memories_dir / f"{slug}.md" if not path.exists(): return False - meta, _ = _parse_frontmatter(path.read_text()) + meta, _ = parse_frontmatter(path.read_text()) path.unlink() self._reindex() self._append_audit( @@ -295,7 +266,7 @@ def _iter_memories(self): parsed = [] for f in files: try: - meta, body = _parse_frontmatter(f.read_text()) + meta, body = parse_frontmatter(f.read_text()) except Exception: continue meta.setdefault("name", f.stem) diff --git a/condor/notifications.py b/condor/notifications.py new file mode 100644 index 000000000..a30506153 --- /dev/null +++ b/condor/notifications.py @@ -0,0 +1,317 @@ +"""Per-user task notifications, and the surfaces they reach (FEAT-048). + +Everything Condor tells a user when a background task finishes — a delegation +completing, a routine run ending, an agent's ``send_notification``, the boot +notice — used to be pushed to Telegram and nowhere else, so a user who only had +the dashboard open learned nothing. This module is the other half: a small +per-user store plus a push bus, so a finished task lights the dashboard's bell +live *and* is still there, unread, an hour later. + +Two rules shape it: + +* **The notice must survive nobody watching.** So a store, not just a push: + ``data/notifications.json``, newest first, capped per user, written with + :func:`condor.fsutil.atomic_write_json` exactly like ``routine_hooks.json``. +* **Core must not import the web layer.** The surface registers itself at + import time, the same shape ``condor/runtime/wake.py`` uses for wake and note + sinks — see ``condor/web/routes/chat_ws.py``. + +:class:`NotifyBot` is the adapter that gets this to every *existing* producer +without touching their call sites: it implements the four methods the codebase's +bot surface actually uses and records instead of sending, so it can sit at the +bottom of ``resolve_bot()``'s ladder and catch anything that has no Telegram to +go to. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import time +import uuid +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Awaitable, Callable + +from condor.fsutil import atomic_write_json + +log = logging.getLogger(__name__) + +_FILE = Path("data") / "notifications.json" + +# Per user. A bell shows the last few dozen; anything older is history nobody +# scrolls to, and an unbounded file would be read and rewritten on every record. +_MAX_PER_USER = 100 + +# Serialises the read-modify-write in :func:`record`. Two producers finishing at +# once would otherwise each read the same file and the second write would drop +# the first's notification. The write rate is a handful per hour, so the lock +# costs nothing and removes the whole class of lost updates. +_write_lock = asyncio.Lock() + + +@dataclass(frozen=True) +class Notification: + """One thing worth telling a user about, addressed to the user.""" + + id: str + user_id: int + ts: float + kind: str # "delegation" | "routine" | "agent" | "system" + text: str + title: str | None = None + # A dashboard route, e.g. "/routines?tab=reports". Relative on purpose: the + # bell navigates within the SPA, it does not open the world. + link: str | None = None + read: bool = False + + def to_wire(self) -> dict[str, Any]: + return asdict(self) + + +# ── The push bus ── + +PushSink = Callable[[Notification], Awaitable[None]] + +_push_sinks: list[PushSink] = [] + + +def register_push_sink(sink: PushSink) -> None: + """Bind a live surface's renderer. Called by the surface, at import time.""" + if sink not in _push_sinks: + _push_sinks.append(sink) + + +async def _push(notification: Notification) -> None: + """Fan out to whoever is watching. A dead sink is not a dead notification. + + Mirrors ``wake._guard``: the store is the record, so a socket that blew up + must never propagate into the producer that was only announcing something. + """ + for sink in list(_push_sinks): + try: + await sink(notification) + except Exception: # noqa: BLE001 - a dead sink is not a dead notification + log.warning("Notification push sink failed", exc_info=True) + + +# ── The store ── + + +def _read_all() -> dict[str, list[dict]]: + if not _FILE.exists(): + return {} + try: + data = json.loads(_FILE.read_text(encoding="utf-8")) + if not isinstance(data, dict): + return {} + return {str(k): v for k, v in data.items() if isinstance(v, list)} + except Exception: # noqa: BLE001 + log.warning("Failed to read %s; treating as empty", _FILE) + return {} + + +def _write_all(data: dict[str, list[dict]]) -> None: + atomic_write_json(_FILE, data, indent=2) + + +def _hydrate(raw: dict) -> Notification | None: + """One stored dict back into a :class:`Notification`, or None if it is junk. + + Tolerant on purpose: a file written by an older build (or by hand) must not + take the whole bell down with it. + """ + try: + return Notification( + id=str(raw["id"]), + user_id=int(raw["user_id"]), + ts=float(raw.get("ts") or 0.0), + kind=str(raw.get("kind") or "system"), + text=str(raw.get("text") or ""), + title=raw.get("title") or None, + link=raw.get("link") or None, + read=bool(raw.get("read")), + ) + except (KeyError, TypeError, ValueError): + return None + + +async def record( + user_id: int | None, + text: str, + *, + kind: str = "system", + title: str | None = None, + link: str | None = None, +) -> Notification | None: + """Store a notification for ``user_id`` and push it to live surfaces. + + Returns the stored notification, or ``None`` when there is nobody to + address it to (a falsy ``user_id``) or nothing to say — a producer with no + user behind it must not be an error, it simply has no bell to ring. + """ + if not user_id or not text: + return None + + notification = Notification( + id=uuid.uuid4().hex, + user_id=int(user_id), + ts=time.time(), + kind=kind, + text=text, + title=title, + link=link, + ) + + async with _write_lock: + data = _read_all() + key = str(notification.user_id) + items = [notification.to_wire()] + (data.get(key) or []) + data[key] = items[:_MAX_PER_USER] + try: + _write_all(data) + except Exception: # noqa: BLE001 - an unwritable store still gets a push + log.warning("Could not persist notification for %s", user_id, exc_info=True) + + await _push(notification) + return notification + + +def list_for(user_id: int, limit: int = 50) -> list[Notification]: + """This user's notifications, newest first. Never anyone else's.""" + if not user_id: + return [] + raw = _read_all().get(str(int(user_id))) or [] + items = [n for n in (_hydrate(r) for r in raw if isinstance(r, dict)) if n] + return items[: max(0, limit)] + + +def unread_count(user_id: int) -> int: + """How many of this user's notifications are still unread.""" + return sum(1 for n in list_for(user_id, limit=_MAX_PER_USER) if not n.read) + + +def mark_read(user_id: int, ids: list[str] | None = None) -> int: + """Mark ``ids`` read (``None`` = all of them). Returns the unread count left. + + Scoped to ``user_id``'s own bucket, so an id belonging to someone else is + simply not found — there is no way to reach across users from here. + """ + if not user_id: + return 0 + key = str(int(user_id)) + data = _read_all() + items = data.get(key) or [] + wanted = set(ids) if ids is not None else None + changed = False + for raw in items: + if not isinstance(raw, dict): + continue + if wanted is not None and str(raw.get("id")) not in wanted: + continue + if not raw.get("read"): + raw["read"] = True + changed = True + if changed: + data[key] = items + try: + _write_all(data) + except Exception: # noqa: BLE001 + log.warning("Could not persist read state for %s", user_id, exc_info=True) + return sum(1 for raw in items if isinstance(raw, dict) and not raw.get("read")) + + +def user_for_chat(chat_id: Any) -> int | None: + """The dashboard user behind a Telegram chat id, or None if there is none. + + A private chat's id *is* the user id — the rule ``_check_chat_access`` + already relies on. A group chat id (negative) has no dashboard owner, and + neither does a private chat with someone this install has never seen, so + both return None: guessing would file one person's message in another + person's bell. + """ + try: + user_id = int(chat_id) + except (TypeError, ValueError): + return None + if user_id <= 0: # Telegram group/channel ids are negative. + return None + try: + from config_manager import get_config_manager + + if get_config_manager().get_user(user_id) is None: + log.debug("No dashboard user for chat %s; not recording", chat_id) + return None + except Exception: # noqa: BLE001 - config unavailable: do not guess + log.debug("Could not resolve chat %s to a user", chat_id, exc_info=True) + return None + return user_id + + +# ── The adapter every existing producer already talks to ── + + +class NotifyBot: + """A bot-shaped object that records notifications instead of sending them. + + The four methods are the ones the codebase's bot surface actually uses — + the same set ``_HttpBot`` implements (``condor/routine_store.py``). Sitting + at the bottom of ``resolve_bot()``'s ladder, this is what makes a routine's + ``context.bot.send_message(...)`` land on the dashboard in an install with + no Telegram token, with no routine and no call-site changes. + + Addressing goes through :func:`user_for_chat`, so nothing is recorded for a + chat id that is not a known user. + + Binary payloads are **not** re-hosted. ``send_photo``/``send_document`` + record the caption (or the filename) and link to the reports browser, which + already serves routine reports through ``GET /reports/{id}/html``. + """ + + def __init__(self, kind: str = "system"): + self._kind = kind + + # -- helpers -- + + @staticmethod + def _arg(a: tuple, kw: dict, name: str, index: int) -> Any: + return kw.get(name) if name in kw else (a[index] if len(a) > index else None) + + async def _record(self, chat_id: Any, text: str, link: str | None = None): + user_id = user_for_chat(chat_id) + if user_id is None: + return None + return await record(user_id, text, kind=self._kind, link=link) + + # -- the bot surface -- + + async def send_message(self, *a, **kw): + chat_id = self._arg(a, kw, "chat_id", 0) + text = self._arg(a, kw, "text", 1) or "" + return await self._record(chat_id, text) + + async def send_photo(self, *a, **kw): + chat_id = self._arg(a, kw, "chat_id", 0) + text = kw.get("caption") or "Chart" + return await self._record(chat_id, text, link="/routines?tab=reports") + + async def send_document(self, *a, **kw): + chat_id = self._arg(a, kw, "chat_id", 0) + document = self._arg(a, kw, "document", 1) + text = ( + kw.get("caption") + or kw.get("filename") + or getattr(document, "name", None) + or "Report" + ) + return await self._record(chat_id, text, link="/routines?tab=reports") + + async def edit_message_text(self, *a, **kw): + """A live-updating message has no bell equivalent: nothing to record. + + The bell is a log of things that happened, not a surface that redraws — + recording every tick of a ``LiveReport`` would bury the notices that + matter under a routine's own progress updates. + """ + return None diff --git a/condor/pool_data.py b/condor/pool_data.py index cb2cf3796..114a4e450 100644 --- a/condor/pool_data.py +++ b/condor/pool_data.py @@ -26,17 +26,9 @@ from glom import glom from condor import orca_api -from condor.cache import evict_expired, get_cached, set_cached -from config_manager import get_client logger = logging.getLogger(__name__) -# The conversation cache these functions write into is shared with the Telegram DEX -# handlers, which read and invalidate it by group under the "_cache" namespace (see -# handlers/dex/_shared). Entries written under any other namespace would survive an -# invalidation the handler thinks it performed, so this must stay in step with it. -_CACHE_NS = "_cache" - # CLMM venues an ``lp_executor`` can open a position on: (brand, the qualifiers its # GeckoTerminal dex id must carry, the gecko networks it exists on). Deliberately # separate from LIQUIDITY_SUPPORTED_DEXES below, which answers a different question @@ -453,6 +445,8 @@ def reset_gecko_throttle() -> None: _token_symbol_cache, _token_pool_cache, _pair_pool_cache, + _ohlcv_cache, + _pool_bins_cache, ): cache.clear() @@ -529,9 +523,8 @@ def _clear(finished: "asyncio.Task", _key: Tuple = key) -> None: # ── Small TTL caches for token lookups ── -# These answers are process-wide (not per-user) and tiny, so they live here rather -# than in a caller's user_data. Capped so an unbounded stream of distinct mints -# cannot grow them without limit. +# These answers are process-wide (not per-user) and tiny. Capped so an unbounded +# stream of distinct mints cannot grow them without limit. _TOKEN_CACHE_MAX = 512 _token_symbol_cache: Dict[Tuple[str, str], Tuple[float, str]] = {} # Both hold normalized pool dicts, and an empty dict for "asked, there is no such @@ -948,22 +941,22 @@ async def fetch_ohlcv( network: str, timeframe: str = "1h", currency: str = "usd", - user_data: dict = None, limit: int = 100, before_timestamp: Optional[int] = None, token: str = "base", + use_cache: bool = True, ) -> Tuple[Optional[List], Optional[str]]: """Fetch OHLCV data for any pool via GeckoTerminal - Concurrent callers asking for the same upstream window share one request, no - matter what key each of them caches the answer under. + A pool's candles are global, not per-user, so the answer is cached once for + the process; concurrent callers asking for the same upstream window share one + request. Args: pool_address: Pool contract address network: Network identifier (will be converted to GeckoTerminal format) timeframe: OHLCV timeframe (see GECKO_TIMEFRAMES) currency: Price currency - "usd" or "token" (quote token) - user_data: Optional user_data dict for caching limit: Number of candles to fetch (capped at GECKO_OHLCV_MAX) before_timestamp: Unix seconds; candles walk back from here. None asks for the latest candles, which is also what keeps a live chart's cache key @@ -971,6 +964,10 @@ async def fetch_ohlcv( token: Which side of the pool to price, "base" or "quote". "quote" flips the series, for a venue pair quoted the other way round from the pool (``XRP-RLUSD`` against a ``RLUSD / XRP`` pool). + use_cache: False skips the cache *read* — for a live poll that must see + the newest candle rather than one up to ``OHLCV_CACHE_TTL`` old. The + fresh answer is still written back, so cached readers only get newer + data out of it. Returns: Tuple of (ohlcv_list, error_message) @@ -982,23 +979,24 @@ async def fetch_ohlcv( timeframe = normalize_timeframe(timeframe) limit = max(1, min(int(limit), GECKO_OHLCV_MAX)) - # Check cache. limit/before_timestamp belong to the key: the same pool - # charted over a historical window and over the live one are different - # answers. - if user_data is not None: - cache_key = ( - f"ohlcv_{gecko_network}_{pool_address}_{timeframe}_{currency}" - f"_{token}_{limit}_{before_timestamp or 0}" - ) - cached = get_cached( - user_data, cache_key, ttl=OHLCV_CACHE_TTL, namespace=_CACHE_NS - ) + # limit/before_timestamp belong to the key: the same pool charted over a + # historical window and over the live one are different answers. The same + # tuple keys the cache and the single-flight, so "one upstream window" is + # one entry everywhere. + cache_key = ( + "ohlcv", + gecko_network, + pool_address, + timeframe, + currency, + token, + limit, + before_timestamp or 0, + ) + if use_cache: + cached = _ttl_get(_ohlcv_cache, cache_key, OHLCV_CACHE_TTL) if cached is not None: return cached, None - # Sweep on miss, as cached_call does. Historical windows mint a fresh - # key per executor, so a long-lived caller would otherwise accumulate - # one entry per chart forever. - evict_expired(user_data, namespace=_CACHE_NS) async def _fetch_upstream() -> Optional[List]: # Pass all parameters explicitly: @@ -1054,34 +1052,17 @@ async def _fetch_upstream() -> Optional[List]: return rows - # Coalesce on the UPSTREAM window — pool, timeframe, currency, priced side - # and window — never on the caller's cache key. The TTL caches above are - # per-caller and only help once an answer has landed, so two callers whose - # own keys differ (a chart keyed by trading pair and one keyed by pool, the - # dashboard's shared cache and a chat's user_data) still resolve to the very - # same GeckoTerminal window and would each spend a request for it. Keyed - # here, they collapse onto one. A fetch that raises reaches every waiter and - # is cached by nobody. The rows are shared between callers: read-only. - ohlcv_list = await _single_flight( - ( - "ohlcv", - gecko_network, - pool_address, - timeframe, - currency, - token, - limit, - before_timestamp or 0, - ), - _fetch_upstream, - ) + # Coalesce on the upstream window: a TTL cache only helps once an answer + # has landed, so concurrent callers of the same window (a cache-bypassing + # live poll racing a chart render, say) would each spend a request without + # this. A fetch that raises reaches every waiter and is cached by nobody. + # The rows are shared between callers: read-only. + ohlcv_list = await _single_flight(cache_key, _fetch_upstream) if not ohlcv_list: return None, "No OHLCV data available" - # Cache result - if user_data is not None: - set_cached(user_data, cache_key, ohlcv_list, namespace=_CACHE_NS) + _ttl_put(_ohlcv_cache, cache_key, ohlcv_list, OHLCV_CACHE_TTL) return ohlcv_list, None @@ -1090,11 +1071,13 @@ async def _fetch_upstream() -> Optional[List]: return None, f"Failed to fetch OHLCV: {str(e)}" -# Bins for one pool, keyed ``(connector, pool_address)``. Used when no caller -# supplies a ``user_data`` to cache in — i.e. by the web dashboard, where every -# viewer of a pool would otherwise open its own Gateway call at the same cadence. -# Same one-minute TTL as the per-chat cache: bins move with every swap through -# the active bin, so a longer one would draw liquidity that has already left. +# OHLCV per upstream window and bins per ``(connector, pool_address)``. Both are +# global answers — a pool's candles and liquidity do not depend on who is asking — +# so they are cached once for the process and every viewer (web dashboard, any +# Telegram chat) shares one upstream call per window. Bins keep a one-minute TTL: +# they move with every swap through the active bin, so a longer one would draw +# liquidity that has already left. +_ohlcv_cache: Dict[Tuple, Tuple[float, List]] = {} _pool_bins_cache: Dict[Tuple[str, str], Tuple[float, Dict[str, Any]]] = {} @@ -1102,9 +1085,6 @@ async def fetch_liquidity_bins( pool_address: str, connector: str = "meteora", network: str = "solana-mainnet-beta", - user_data: dict = None, - chat_id: int = None, - context=None, client=None, ) -> Tuple[Optional[List], Optional[Dict], Optional[str]]: """Fetch liquidity bin data for CLMM pools via gateway @@ -1113,13 +1093,10 @@ async def fetch_liquidity_bins( pool_address: Pool contract address connector: DEX connector (meteora, raydium, orca) network: Network identifier - user_data: Optional user_data dict for caching. When omitted the result - is cached process-wide instead, so callers with no conversation of - their own (the web dashboard) still share one Gateway call per pool. - chat_id: Chat ID for per-chat server selection - client: An already-resolved API client. The web side holds a server - *name* and resolves its client through the config manager, which the - ``get_client(chat_id)`` path below cannot express. + client: An already-resolved API client, required to reach Gateway. + Resolving one is the caller's job — the web side holds a server + *name*, a Telegram handler a chat id, and each has its own accessor + for turning that into a client. Returns: Tuple of (bins_list, pool_info, error_message) @@ -1131,21 +1108,10 @@ async def fetch_liquidity_bins( if not can_fetch_liquidity(connector): return None, None, f"Liquidity data not available for {connector}" - # Check cache - cache_key = f"pool_bins_{connector}_{pool_address}" - if user_data is not None: - cached = get_cached( - user_data, cache_key, ttl=BINS_CACHE_TTL, namespace=_CACHE_NS - ) - else: - cached = _ttl_get( - _pool_bins_cache, (connector, pool_address), BINS_CACHE_TTL - ) + cached = _ttl_get(_pool_bins_cache, (connector, pool_address), BINS_CACHE_TTL) if cached is not None: return cached.get("bins"), cached, None - if client is None: - client = await get_client(chat_id, context=context) if not client: return None, None, "Gateway client not available" @@ -1205,13 +1171,7 @@ async def fetch_liquidity_bins( bins = pool_info.get("bins", []) - # Cache result - if user_data is not None: - set_cached(user_data, cache_key, pool_info, namespace=_CACHE_NS) - else: - _ttl_put( - _pool_bins_cache, (connector, pool_address), pool_info, BINS_CACHE_TTL - ) + _ttl_put(_pool_bins_cache, (connector, pool_address), pool_info, BINS_CACHE_TTL) return bins, pool_info, None diff --git a/condor/preferences.py b/condor/preferences.py index 73ac5b9c7..f11cb425c 100644 --- a/condor/preferences.py +++ b/condor/preferences.py @@ -14,8 +14,9 @@ - Primary: config.yml via ConfigManager (shared across TG + Web) - Fallback: context.user_data pickle (for session-level state) -When a user_data dict contains '_user_id', changes are synced to ConfigManager -so the web dashboard can also read them. +When a user_data dict contains '_user_id', setters sync the whole affected +preference section to ConfigManager (via _sync_section_to_cm) so the web +dashboard can also read them. """ import logging @@ -37,20 +38,6 @@ def _get_user_id(user_data: Dict) -> Optional[int]: return user_data.get("_user_id") -def _sync_to_cm(user_data: Dict, key: str, value) -> None: - """Persist a preference to ConfigManager (config.yml) if user_id is known.""" - user_id = _get_user_id(user_data) - if user_id is None: - return - try: - from config_manager import get_config_manager - - cm = get_config_manager() - cm.set_user_preference(user_id, key, value) - except Exception as e: - logger.debug("Failed to sync preference '%s' to config: %s", key, e) - - def _sync_section_to_cm(user_data: Dict, section: str) -> None: """Persist an entire preference section to ConfigManager.""" user_id = _get_user_id(user_data) @@ -105,6 +92,12 @@ def _load_from_cm(user_data: Dict) -> None: USER_PREFERENCES_KEY = "user_preferences" _MIGRATION_DONE_KEY = "_prefs_migration_done" +# Marks a ``user_data`` built *for* one server — a routine or agent run launched +# against it — rather than a chat's own preferences. ``get_effective_server`` +# honours a pinned server outright; without the mark it treats ``active_server`` +# as a cache of the chat default and lets config.yml correct it. +SERVER_PIN_KEY = "_server_pinned" + # Portfolio defaults DEFAULT_PORTFOLIO_DAYS = 3 DEFAULT_PORTFOLIO_INTERVAL = "1h" @@ -189,16 +182,6 @@ class GeneralPrefs(TypedDict, total=False): active_server: Optional[str] -class WalletNetworkPrefs(TypedDict, total=False): - """Network preferences for a specific wallet. - - Keys are wallet addresses, values are lists of enabled network IDs. - Example: {"0x1234...": ["ethereum-mainnet", "base", "arbitrum"]} - """ - - pass # Dynamic keys based on wallet addresses - - class GatewayPrefs(TypedDict, total=False): """Gateway-related preferences including wallet network settings.""" @@ -567,16 +550,6 @@ def get_general_prefs(user_data: Dict) -> GeneralPrefs: # ============================================ -def get_portfolio_days(user_data: Dict) -> int: - """Get portfolio graph days setting""" - return get_portfolio_prefs(user_data).get("days", DEFAULT_PORTFOLIO_DAYS) - - -def get_portfolio_interval(user_data: Dict) -> str: - """Get portfolio graph interval setting""" - return get_portfolio_prefs(user_data).get("interval", DEFAULT_PORTFOLIO_INTERVAL) - - def set_portfolio_days(user_data: Dict, days: int) -> None: """Set portfolio graph days""" prefs = _ensure_preferences(user_data) @@ -585,14 +558,6 @@ def set_portfolio_days(user_data: Dict, days: int) -> None: logger.info(f"Set portfolio days to {days}") -def set_portfolio_interval(user_data: Dict, interval: str) -> None: - """Set portfolio graph interval""" - prefs = _ensure_preferences(user_data) - prefs["portfolio"]["interval"] = interval - _sync_section_to_cm(user_data, "portfolio") - logger.info(f"Set portfolio interval to {interval}") - - # ============================================ # PUBLIC API - CLOB TRADING # ============================================ @@ -603,19 +568,6 @@ def get_clob_account(user_data: Dict) -> str: return get_clob_prefs(user_data).get("account", DEFAULT_CLOB_ACCOUNT) -def set_clob_account(user_data: Dict, account: str) -> None: - """Set CLOB trading account""" - prefs = _ensure_preferences(user_data) - prefs["clob"]["account"] = account - _sync_section_to_cm(user_data, "clob") - logger.info(f"Set CLOB account to {account}") - - -def get_clob_last_order(user_data: Dict) -> CLOBOrderParams: - """Get last CLOB order parameters""" - return deepcopy(get_clob_prefs(user_data).get("last_order", {})) - - def set_clob_last_order(user_data: Dict, params: CLOBOrderParams) -> None: """Set last CLOB order parameters (for quick trading)""" prefs = _ensure_preferences(user_data) @@ -683,19 +635,6 @@ def get_dex_connector(user_data: Dict, network: Optional[str] = None) -> str: return get_dex_prefs(user_data).get("default_connector", DEFAULT_DEX_CONNECTOR) -def get_dex_slippage(user_data: Dict) -> str: - """Get default DEX slippage percentage""" - return get_dex_prefs(user_data).get("default_slippage", DEFAULT_DEX_SLIPPAGE) - - -def set_dex_slippage(user_data: Dict, slippage: str) -> None: - """Set default DEX slippage percentage""" - prefs = _ensure_preferences(user_data) - prefs["dex"]["default_slippage"] = slippage - _sync_section_to_cm(user_data, "dex") - logger.info(f"Set DEX slippage to {slippage}%") - - def get_dex_last_swap(user_data: Dict) -> DEXSwapParams: """Get last DEX swap parameters""" return deepcopy(get_dex_prefs(user_data).get("last_swap", {})) @@ -975,12 +914,6 @@ def get_executor_deployed_pairs(user_data: Dict) -> List[str]: return get_executor_prefs(user_data).get("deployed_pairs", []) -def set_executor_deployed_pairs(user_data: Dict, pairs: List[str]) -> None: - """Set recently deployed trading pairs""" - prefs = _ensure_preferences(user_data) - prefs["executors"]["deployed_pairs"] = pairs[:8] - - def add_executor_deployed_pair(user_data: Dict, pair: str) -> None: """Add a trading pair to the front of the deployed pairs list""" prefs = _ensure_preferences(user_data) @@ -1047,16 +980,6 @@ def get_agent_prefs(user_data: Dict) -> "AgentPrefs": ) -def set_default_agent(user_data: Dict, agent_key: str) -> None: - """Set default agent""" - prefs = _ensure_preferences(user_data) - if "agent" not in prefs: - prefs["agent"] = {} - prefs["agent"]["default_agent"] = agent_key - _sync_section_to_cm(user_data, "agent") - logger.info(f"Set default agent to {agent_key}") - - def get_chat_binding(user_data: Dict) -> "ChatBindingPrefs": """What this chat is bound to — see :class:`ChatBindingPrefs`. @@ -1082,15 +1005,6 @@ def set_chat_binding(user_data: Dict, binding: "ChatBindingPrefs") -> None: _sync_section_to_cm(user_data, "agent") -def clear_chat_binding(user_data: Dict) -> None: - """Forget the binding, so later respawns come back unbound (Condor).""" - prefs = _ensure_preferences(user_data) - agent = prefs.get("agent") or {} - if agent.pop("chat_binding", None) is None: - return - _sync_section_to_cm(user_data, "agent") - - # ============================================ # PUBLIC API - CUSTOM OPENAI-COMPATIBLE PROVIDERS # ============================================ @@ -1237,6 +1151,7 @@ def resolve_custom_endpoint( agent_key: str, user_data: Optional[Dict] = None, user_id: Optional[int] = None, + strict: bool = False, ) -> tuple[Optional[str], Optional[str]]: """Resolve ``(base_url, api_key)`` for a ``custom@:`` key. @@ -1250,6 +1165,12 @@ def resolve_custom_endpoint( Pass ``user_data`` when you have it (Telegram handlers) or ``user_id`` when you don't (MCP subprocess, web, background agent runs). + + ``strict=True`` raises a :class:`RuntimeError` with an actionable message + when the key *names* an endpoint that isn't saved, instead of logging a + warning and falling back to the ``CUSTOM_LLM_*`` env vars. Interactive + surfaces (chat sessions) want the loud failure; background surfaces + (consult, engine) keep the lenient default. """ provider_name, model_id = parse_custom_agent_key(agent_key) if not model_id: @@ -1263,6 +1184,12 @@ def resolve_custom_endpoint( provider = find_custom_provider(user_data, provider_name) if user_data else None if provider is None and provider_name: + if strict: + raise RuntimeError( + f"No saved endpoint named '{provider_name}'. Add it via " + "/agent → Change LLM → Custom endpoint, or Settings → " + "AI Providers on the web dashboard." + ) logger.warning( "Agent key '%s' names endpoint '%s', which is not saved for this user", agent_key, @@ -1400,18 +1327,6 @@ def get_voice_prefs(user_data: Dict) -> "VoicePrefs": ) -def set_voice_prefs(user_data: Dict, **kwargs) -> None: - """Update voice preferences. Pass only the keys you want to change.""" - prefs = _ensure_preferences(user_data) - if "voice" not in prefs: - prefs["voice"] = {"whisper_model": "small", "language": None, "auto_send": True} - for key in ("whisper_model", "language", "auto_send"): - if key in kwargs: - prefs["voice"][key] = kwargs[key] - _sync_section_to_cm(user_data, "voice") - logger.info("Updated voice preferences: %s", kwargs) - - # ============================================ # PUBLIC API - NOTES (key-value memory for the AI agent) # ============================================ @@ -1438,17 +1353,6 @@ def set_note(user_data: Dict, key: str, value: str) -> None: logger.info(f"Set note '{key}'") -def delete_note(user_data: Dict, key: str) -> bool: - """Delete a note by key. Returns True if it existed.""" - prefs = _ensure_preferences(user_data) - notes = prefs.get("notes", {}) - if key in notes: - del notes[key] - logger.info(f"Deleted note '{key}'") - return True - return False - - # ============================================ # UTILITY FUNCTIONS # ============================================ @@ -1462,8 +1366,3 @@ def clear_preferences(user_data: Dict) -> None: user_data.pop(_MIGRATION_DONE_KEY, None) user_data.pop("_prefs_hydrated", None) logger.info("Cleared all user preferences") - - -def export_preferences(user_data: Dict) -> Dict[str, Any]: - """Export all preferences as a dictionary (for debugging/backup)""" - return get_preferences(user_data) diff --git a/condor/reports/__init__.py b/condor/reports/__init__.py index 31fa4d3e9..e8001c7c1 100644 --- a/condor/reports/__init__.py +++ b/condor/reports/__init__.py @@ -5,6 +5,7 @@ CHARTS_DIR, INDEX_FILE, MAX_REPORTS, + attribute_owner, attribute_to, default_source, delete_report, @@ -22,6 +23,7 @@ "CHARTS_DIR", "INDEX_FILE", "MAX_REPORTS", + "attribute_owner", "attribute_to", "default_source", "reset_last_report_id", diff --git a/condor/reports/builder.py b/condor/reports/builder.py index 7220b4687..cabb84ff8 100644 --- a/condor/reports/builder.py +++ b/condor/reports/builder.py @@ -460,6 +460,11 @@ async def save(self, report_id: str | None = None) -> str: "source_name": source_name, "tags": self._tags, "agent": store._report_agent.get() or "condor", + # The authenticated principal the run executes for (SEC-196): + # every runner wraps execution in store.attribute_owner, and the + # web routes authorize reads/deletes against this id. None (no + # wrapper, e.g. a bare script) makes the report admin-only. + "user_id": store._report_owner.get(), } entries = store._read_index() entries.append(entry) diff --git a/condor/reports/store.py b/condor/reports/store.py index 7d00e2956..f796a7b66 100644 --- a/condor/reports/store.py +++ b/condor/reports/store.py @@ -26,6 +26,9 @@ _report_source: contextvars.ContextVar[tuple[str, str] | None] = contextvars.ContextVar( "report_source", default=None ) +_report_owner: contextvars.ContextVar[int | None] = contextvars.ContextVar( + "report_owner", default=None +) def _charts_dir() -> Path: @@ -62,6 +65,23 @@ def attribute_to(agent: str | None): _report_agent.reset(token) +@contextmanager +def attribute_owner(user_id: int | None): + """Attribute reports saved within this block to an authenticated user id. + + Every runner (routine store, code runner, Telegram routine handler, agent + session report) wraps execution with this so ``ReportBuilder.save`` records + who the report belongs to — the id the web routes then authorize reads and + deletes against (SEC-196). A falsy id (0, None) records no owner, which the + routes treat as admin-only, never world-readable. + """ + token = _report_owner.set(int(user_id) if user_id else None) + try: + yield + finally: + _report_owner.reset(token) + + @contextmanager def default_source(source_type: str, source_name: str): """Stamp reports saved in this block that never called ``ReportBuilder.source()``. @@ -128,10 +148,21 @@ def list_reports( agent: str | None = None, limit: int = 50, offset: int = 0, + owner_id: int | None = None, ) -> tuple[list[dict], int]: + """List index entries, newest first. + + ``owner_id`` scopes the listing to one user's reports: entries whose + ``user_id`` differs — including legacy entries with no owner at all — + are dropped (fail closed, SEC-196). ``None`` means no owner filter, + which the web routes reserve for admins; internal callers that match + on ``source_name`` (routine run lists) also pass ``None``. + """ entries = _read_index() entries.sort(key=lambda entry: entry.get("created_at", ""), reverse=True) + if owner_id is not None: + entries = [entry for entry in entries if entry.get("user_id") == owner_id] if source_type: entries = [ entry for entry in entries if entry.get("source_type") == source_type @@ -154,10 +185,16 @@ def list_reports( return entries[offset : offset + limit], total -def list_reports_grouped() -> list[dict]: - """Return the latest report per source name, with count.""" +def list_reports_grouped(owner_id: int | None = None) -> list[dict]: + """Return the latest report per source name, with count. + + ``owner_id`` scopes the grouping exactly as in ``list_reports``: ``None`` + (admin) sees every entry, anyone else only entries stamped with their id. + """ entries = _read_index() entries.sort(key=lambda entry: entry.get("created_at", ""), reverse=True) + if owner_id is not None: + entries = [entry for entry in entries if entry.get("user_id") == owner_id] groups: dict[str, dict] = {} for entry in entries: source_name = entry.get("source_name", "") diff --git a/condor/routine_store.py b/condor/routine_store.py index 4bb04491f..5a5433bea 100644 --- a/condor/routine_store.py +++ b/condor/routine_store.py @@ -94,6 +94,23 @@ async def edit_message_text(self, *a, **kw): data = {k: v for k, v in kw.items() if v is not None} return await self._post("editMessageText", data) + def has_token(self) -> bool: + """Can this actually deliver? Without a token every send is a no-op. + + Read by ``delegate.resolve_bot`` so the ladder can fall through to the + dashboard bell rather than to a sender that silently drops the message. + """ + return bool(self._token) + + async def get_chat_member(self, *a, **kw): + """Raw ``getChatMember`` envelope — the web routes' chat-ownership check + (SEC-198) reads ``result.status`` out of it when no live bot is around.""" + chat_id = kw.get("chat_id") or (a[0] if a else None) + user_id = kw.get("user_id") or (a[1] if len(a) > 1 else None) + return await self._post( + "getChatMember", {"chat_id": chat_id, "user_id": user_id} + ) + _http_bot = _HttpBot() @@ -125,7 +142,7 @@ class WebRoutineContext: """Lightweight context so routines can run without Telegram.""" def __init__(self, server_name: str, bot=None, chat_id: int = 0): - from condor.preferences import USER_PREFERENCES_KEY + from condor.preferences import SERVER_PIN_KEY, USER_PREFERENCES_KEY self._chat_id = chat_id # The server this run was launched against, readable by name — for a routine @@ -137,8 +154,12 @@ def __init__(self, server_name: str, bot=None, chat_id: int = 0): # "preferences" while the preference API reads USER_PREFERENCES_KEY # ("user_preferences"), so the server a web or agent run was launched with # never reached its client — every such run silently picked the chat default. + # Pinned, not preferred: this run was launched against `server_name`, so + # the chat's own default must not override it — the chat may well be a + # chat whose default is a different server (see `get_effective_server`). self._user_data: dict[str, Any] = { USER_PREFERENCES_KEY: {"general": {"active_server": server_name}}, + SERVER_PIN_KEY: True, } @property @@ -428,17 +449,35 @@ async def _report_run( worth showing, not worth a model turn to announce it. A run with no conversation behind it — the scheduler, the dashboard, the - Telegram menu, an instance restored on boot — is a no-op: ``record_system`` - ignores an empty id and a run with no session key reaches no surface. - Neither delivery is allowed to raise: a missing note must not cost the - user the run's own result or its hooks. + Telegram menu, an instance restored on boot — still lights the owner's + bell (FEAT-048) and is otherwise a no-op: ``record_system`` ignores an + empty id and a run with no session key reaches no surface. No delivery + is allowed to raise: a missing note must not cost the user the run's own + result or its hooks. """ meta = self._instances.get(instance_id) or {} + text = _run_outcome_text(meta.get("routine_name") or "", summary, error) + + # The bell is addressed to the *owner*, not to a conversation, so it is + # the one surface a scheduled or dashboard-started run can reach. + try: + from condor.notifications import record + + await record( + meta.get("user_id"), + text, + kind="routine", + link="/routines?tab=reports", + ) + except Exception: + logger.debug( + f"Could not notify owner about run {instance_id}", exc_info=True + ) + conversation_id = meta.get("conversation_id") or "" if not conversation_id: return - text = _run_outcome_text(meta.get("routine_name") or "", summary, error) try: from condor.runtime.conversations import record_system @@ -543,9 +582,12 @@ async def _execute_and_record( # The bare name (not "agent_slug/name") is what both report lookups # match on, and what routines that do call .source() already use. base_name = (routine.name or "").split("/")[-1] - with reports.attribute_to(agent or _agent_of(routine)): - with reports.default_source("routine", base_name): - raw = await routine.run_fn(cfg, ctx) + # attribute_owner records who this run executes for, so its reports + # are readable by (and only by) that user on the web (SEC-196). + with reports.attribute_owner(user_id): + with reports.attribute_to(agent or _agent_of(routine)): + with reports.default_source("routine", base_name): + raw = await routine.run_fn(cfg, ctx) result = normalize_result(raw) except asyncio.CancelledError: result = RoutineResult(text="Stopped by user") diff --git a/condor/runtime/binding.py b/condor/runtime/binding.py index c4113e465..8d2388494 100644 --- a/condor/runtime/binding.py +++ b/condor/runtime/binding.py @@ -17,7 +17,9 @@ import logging from dataclasses import dataclass, field +from condor.llm import options as llm_options from condor.memory.paths import CHAT_SLUG +from condor.runtime import toolsets from condor.runtime.models import SessionSpec log = logging.getLogger(__name__) @@ -86,7 +88,6 @@ def resolve( Callers get a fully resolved toolset and identity and never re-derive either. """ from condor.agents.agent import Agent, AgentStore - from handlers.agents._shared import build_mcp_servers_for_session slug = spec.agent_slug or CHAT_SLUG agent = AgentStore().get(slug) @@ -116,7 +117,7 @@ def resolve( # in CONDOR_USER_ID/CONDOR_CHAT_ID, and argv beats env in the subprocess, so # a local fallback here would silently override the env one (SEC-180). effective_user_id, effective_chat_id = spec.effective_ids() - mcp_servers = build_mcp_servers_for_session( + mcp_servers = toolsets.build_mcp_servers_for_session( effective_user_id, effective_chat_id, user_data, @@ -153,13 +154,11 @@ def remember_model_choice(user_id: int | None, agent_slug: str, agent_key: str) delegate and loops pass concrete keys for their own plumbing reasons and must not rewrite anything. """ - from handlers.agents._shared import AGENT_OPTIONS - if not agent_key or not user_id: return # Drill-downs, not startable models: an agent_key of "openrouter:" would # fail at session start for everyone who inherited it. - if AGENT_OPTIONS.get(agent_key, {}).get("picker"): + if llm_options.AGENT_OPTIONS.get(agent_key, {}).get("picker"): return try: diff --git a/condor/runtime/channels.py b/condor/runtime/channels.py new file mode 100644 index 000000000..6869b8a5b --- /dev/null +++ b/condor/runtime/channels.py @@ -0,0 +1,48 @@ +"""Delivery channels for pending trade confirmations (ARCH-190). + +The pending approval itself lives in :mod:`condor.runtime.confirmations`; a +channel only renders it somewhere a human can answer. ``TelegramChannel`` moved +here from ``handlers/agents/confirmation.py`` so the runtime (consult, +delegate) no longer imports from the hot-reloaded handlers package; that module +re-exports it for its Telegram callers. +""" + +from __future__ import annotations + +from telegram import Bot, InlineKeyboardButton, InlineKeyboardMarkup + +from condor.runtime.confirmations import PendingConfirmation + + +class TelegramChannel: + """Renders a pending confirmation as an inline keyboard in a chat.""" + + def __init__(self, bot: Bot, chat_id: int): + self._bot = bot + self._chat_id = chat_id + + async def deliver(self, pending: PendingConfirmation) -> None: + keyboard = InlineKeyboardMarkup( + [ + [ + InlineKeyboardButton( + "Approve", callback_data=f"agent:confirm_trade:{pending.id}" + ), + InlineKeyboardButton( + "Reject", callback_data=f"agent:reject_trade:{pending.id}" + ), + ] + ] + ) + # Naming the asker is not decoration: one chat can talk to several + # agents on several servers, and "Approve this action?" on its own does + # not say whose action, or where it lands. + asked_by = f"\nRequested by {pending.origin}\n" if pending.origin else "" + await self._bot.send_message( + chat_id=self._chat_id, + text=( + f"Trade Confirmation\n{asked_by}\n" + f"{pending.summary}\n\nApprove this action?" + ), + reply_markup=keyboard, + ) diff --git a/condor/runtime/confirmations.py b/condor/runtime/confirmations.py index d18821e6e..01d3fc545 100644 --- a/condor/runtime/confirmations.py +++ b/condor/runtime/confirmations.py @@ -25,6 +25,7 @@ from enum import Enum from typing import Any, Protocol +from condor.runtime import danger from condor.runtime.timeouts import TIMEOUTS from condor.telemetry import taps as telemetry_taps @@ -352,21 +353,19 @@ def build_permission_callback( async def permission_callback( tool_call: dict[str, Any], options: list[dict[str, Any]] ) -> dict[str, Any]: - from handlers.agents._shared import BLOCKED_TOOLS, is_dangerous_tool_call - from handlers.agents.confirmation import format_tool_summary - tool_name = tool_call.get("tool", "") or tool_call.get("title", "") - if tool_name in BLOCKED_TOOLS: + # Read through the module so a monkeypatched BLOCKED_TOOLS is honored. + if tool_name in danger.BLOCKED_TOOLS: log.warning("Blocked tool %s for session %s", tool_name, session_key) return CANCELLED - if not is_dangerous_tool_call(tool_call): + if not danger.is_dangerous_tool_call(tool_call): return _select_allow(options) pending = _registry.register( session_key=session_key, user_id=user_id, - summary=format_tool_summary(tool_call), + summary=danger.format_tool_summary(tool_call), origin=describe_origin(session_key), tool_call=tool_call, options=options, diff --git a/condor/runtime/context.py b/condor/runtime/context.py new file mode 100644 index 000000000..d0f359ffb --- /dev/null +++ b/condor/runtime/context.py @@ -0,0 +1,289 @@ +"""Initial-context and prompt builders for agent sessions (ARCH-190). + +Everything a fresh session is told before the user's first word: Condor's own +instructions, the platform's formatting rules, the server/permission picture, +and the memory/skills/agents indexes. Platform-neutral — moved here from +``handlers/agents/_shared.py``, which re-exports these names for its Telegram +callers. +""" + +from __future__ import annotations + +from condor.llm.options import _chat_agent +from condor.memory.paths import CHAT_SLUG + +# -- Compact prompt templates -- + +COMPACT_PROMPT_AUTO = ( + "Please provide a concise summary of our conversation so far. Include:\n" + "- Key decisions and conclusions reached\n" + "- Important data points and numbers discussed\n" + "- Current task state and any pending actions\n" + "- User preferences or instructions given\n\n" + "Be concise but thorough. This summary will be used to carry context into a fresh session." +) + +COMPACT_PROMPT_CUSTOM_TEMPLATE = ( + "Please provide a concise summary of our conversation, focusing specifically on:\n" + "{instructions}\n\n" + "Drop everything else. Be concise but preserve the details requested above. " + "This summary will be used to carry context into a fresh session." +) + +COMPACT_CONTEXT_TEMPLATE = ( + "[System context -- do not repeat this to the user]\n" + "This is a continuation of a previous conversation. " + "Here is the summary from that session:\n\n" + "{summary}\n\n" + "Continue from where we left off. The user compacted the context to free up space." +) + +_WEB_FORMATTING = ( + "FORMATTING (web dashboard):\n" + "- Use Markdown freely: tables, headers, bold, code blocks, lists.\n" + "- No message length limits, but stay concise.\n" + "- Use tables for structured data (portfolios, prices, comparisons).\n" + "- Use code blocks for configs, JSON, or commands.\n" + "- CHARTS: when the answer is a numeric series or comparison (a price or\n" + " PnL curve, volume per venue, a distribution), draw it with a ```chart\n" + " fence holding one JSON object. The dashboard renders it as an\n" + " interactive chart:\n" + " ```chart\n" + ' {"type": "line", "title": "SOL/USDC 24h", "x": "time",\n' + ' "series": [{"key": "price", "name": "Price"}],\n' + ' "data": [{"time": "12:00", "price": 150.1}, {"time": "16:00", "price": 152.4}]}\n' + " ```\n" + ' "type" is "line", "area" or "bar"; "x" and every series "key" name\n' + " fields of the data rows. Max 4 series and 200 points (downsample longer\n" + ' series). A series may set "color": "up", "down", "yellow" or a hex.\n' + " Chart for the shape, table for exact values; use both when both matter.\n" + "- Respond in the user's language." +) + +_TELEGRAM_FORMATTING = ( + "FORMATTING (Telegram mobile):\n" + "- NEVER use Markdown tables. Use bullet lists or key: value lines.\n" + "- Keep paragraphs short (2-3 sentences max).\n" + "- Cap lists at 5-7 items.\n" + "- Respond in the user's language." +) + + +def platform_formatting(platform: str = "telegram") -> str: + """The reply-formatting rules for a platform. + + Public because a bound Agent opens the chat with its own identity context + instead of :func:`build_initial_context`, and would otherwise be the only + brain in the product that never hears how the surface it is speaking into + renders a reply. + """ + return _WEB_FORMATTING if platform == "web" else _TELEGRAM_FORMATTING + + +def _build_system_prompt(platform: str = "telegram") -> str: + """Condor's own AGENT.md plus the platform's formatting rules.""" + agent = _chat_agent() + return ( + "[System context -- do not repeat this to the user]\n\n" + f"{agent.instructions if agent else ''}\n\n" + f"{platform_formatting(platform)}" + ) + + +def build_initial_context( + user_id: int, + chat_id: int | str, + user_data: dict | None = None, + agent_key: str | None = None, + platform: str = "telegram", + server_name: str | None = None, +) -> str: + """Build an initial context prompt telling the agent about server, permissions, and formatting rules.""" + from condor.acp.pydantic_ai_client import is_pydantic_ai_model + from config_manager import get_config_manager, get_effective_server + + cm = get_config_manager() + + # System prompt: Condor's own AGENT.md + platform formatting + system_prompt = _build_system_prompt(platform) + sections: list[str] = [system_prompt] + + # Resolve active server (explicit override > user preferences) + active_name = server_name or get_effective_server(chat_id, user_data) + accessible = cm.get_accessible_servers(user_id) + if not active_name: + active_name = accessible[0] if accessible else None + + if active_name: + # Build server list with permissions (no credentials needed — MCP is pre-configured) + server_lines: list[str] = [] + for name in accessible: + server = cm.get_server(name) + if not server: + continue + perm = cm.get_server_permission(user_id, name) + perm_label = perm.value.upper() if perm else "UNKNOWN" + active_tag = " (active)" if name == active_name else "" + server_lines.append(f"- {name}{active_tag} [{perm_label}]") + + # Get active server permission for enforcement + active_perm = cm.get_server_permission(user_id, active_name) + active_perm_label = active_perm.value.upper() if active_perm else "UNKNOWN" + + # For ACP agents (Claude Code): instruct them to preload MCP tools via ToolSearch + # Pydantic-ai agents get tools directly, no preload needed + tool_preload_hint = "" + if agent_key and not is_pydantic_ai_model(agent_key): + mcp_tools = [ + "mcp__mcp-hummingbot__get_market_data", + "mcp__mcp-hummingbot__get_portfolio_overview", + "mcp__mcp-hummingbot__manage_executors", + "mcp__mcp-hummingbot__manage_bots", + "mcp__mcp-hummingbot__manage_controllers", + "mcp__mcp-hummingbot__explore_dex_pools", + "mcp__mcp-hummingbot__explore_geckoterminal", + "mcp__mcp-hummingbot__manage_gateway_swaps", + "mcp__mcp-hummingbot__manage_gateway_config", + "mcp__mcp-hummingbot__manage_gateway_container", + "mcp__mcp-hummingbot__search_history", + "mcp__mcp-hummingbot__set_account_position_mode_and_leverage", + "mcp__condor__manage_routines", + "mcp__condor__manage_servers", + "mcp__condor__get_user_context", + "mcp__condor__manage_trading_agent", + "mcp__condor__trading_agent_journal_read", + "mcp__condor__trading_agent_journal_write", + "mcp__condor__send_notification", + "mcp__condor__manage_memory", + "mcp__condor__manage_skill", + ] + tool_preload_hint = ( + "IMPORTANT: At the very start of the session (before your first response), " + "load ALL MCP tools in a single ToolSearch call:\n" + f'ToolSearch(query="select:{",".join(mcp_tools)}")\n' + "This avoids repeated ToolSearch calls that waste context tokens. " + "Do this silently without telling the user." + ) + + # Build server info section + server_info = [ + f"Active server: {active_name} (permission: {active_perm_label})", + "The MCP server is already connected to this server. Do NOT call configure_server — it is pre-configured.", + "", + ] + + if tool_preload_hint: + server_info.extend([tool_preload_hint, ""]) + + server_info.extend( + [ + "Available servers:", + *server_lines, + "", + "Server switching is not supported mid-session. If the user wants a different server, " + "tell them to start a new session with that server selected.", + "", + "Permission rules:", + f"- Your current permission on '{active_name}' is {active_perm_label}.", + "- OWNER: Full access including trading operations and server management.", + "- TRADER: Can trade, view balances, and manage own settings.", + "- Enforce the permission level for this server.", + ] + ) + + sections.append("\n".join(server_info)) + + # User memory index — what the chat assistant remembers about this user. This + # store is the chat's own (FEAT-003), not shared with the user's trading + # agents. Inject only the index; bodies are read on demand via + # manage_memory(action="read"). Nothing injected for new users. + try: + from condor.memory import MemoryStore + + memory_index = MemoryStore(user_id).list_index() + if memory_index: + sections.append( + "[USER MEMORY — what you remember about this user]\n" + "Consider this before responding. Read a full memory with " + 'manage_memory(action="read", name="..."). When you learn something ' + 'new and stable about the user, save it with manage_memory(action="write", ...).\n\n' + f"{memory_index}" + ) + except Exception: + pass # Memory is advisory — never block session start on it. + + # Skills index — read-only playbooks the chat assistant ships (know-how + + # steps). Skills are general to the assistant, not learned per user; inject + # only the index, bodies are read on demand via manage_skill(action="read"). + # Nothing injected when the assistant ships none. + try: + from condor.memory import SkillStore + + skills_index = SkillStore().list_index() + if skills_index: + sections.append( + "[SKILLS — check here BEFORE handling a known flow with raw tools]\n" + "Read-only playbooks shipped with the assistant. Before using raw " + "tools for a flow below, read its playbook with " + 'manage_skill(action="read", name="...") and follow the steps — ' + "don't re-derive or hand-roll what a playbook already covers.\n\n" + f"{skills_index}" + ) + except Exception: + pass # Skills are advisory — never block session start on them. + + # Agents index — domain Agents condor can consult (FEAT: coordinator model). + # condor delegates domain work via consult(...) instead of holding the domain's + # tools/context itself. EVERY agent is listed: one missing from this index can + # never be routed to. Nothing injected only when none exist. + try: + from condor.agents.agent import AgentStore + + # The coordinator is not among its own consultees (FEAT-033). + agents_index = AgentStore().list_index(exclude={CHAT_SLUG}) + if agents_index: + sections.append( + "[AGENTS — consult these BEFORE doing domain work with raw tools]\n" + "You are a coordinator. If a request falls in an agent's domain " + "below, delegate it with " + 'consult(agent="", task="...", context="...") instead of ' + "driving the domain's raw tools yourself — the agent has the focused " + "tools and domain memory. Relay a concise summary of its answer.\n\n" + f"{agents_index}" + ) + except Exception: + pass # The agents index is advisory — never block session start on it. + + return "\n\n".join(sections) + + +def build_agent_context( + agent: "Agent", # noqa: F821 — condor.agents.agent.Agent + user_id: int, + task: str, + context: str = "", +) -> str: + """Assemble the prompt for an Agent consult. + + Mirrors :func:`build_initial_context` but for a worker run: the Agent's own + instructions become the system prompt, its domain-scoped memory/skills indexes + are injected (keyed by the Agent slug, FEAT-003 — the shared brain), and the + consult task is appended last. Read on demand via manage_memory/manage_skill + inside the run. + + Those indexes come from :func:`~condor.memory.domain_context`, the same + builder ``binding.agent_identity_context`` composes, so this Agent carries + identical instructions about its own memory whether it is consulted or + chatted with (ARCH-099). + """ + from condor.memory import domain_context + + sections: list[str] = [agent.instructions] + sections.extend(domain_context(agent.slug, user_id)) + + consult = f"[CONSULT REQUEST]\n{task}" + if context: + consult += f"\n\n[CONTEXT FROM CONDOR]\n{context}" + sections.append(consult) + + return "\n\n".join(sections) diff --git a/condor/runtime/danger.py b/condor/runtime/danger.py new file mode 100644 index 000000000..fc2a7540a --- /dev/null +++ b/condor/runtime/danger.py @@ -0,0 +1,198 @@ +"""Security-critical tool-call classification (ARCH-190). + +Which tool calls need a human, which are refused outright, and how a pending +call is summarized for the confirmation prompt. This used to live in the +Telegram package (``handlers/agents/_shared.py`` / ``confirmation.py``), which +``main.py`` hot-reloads via watchfiles — so the runtime's trade gate depended +on reload timing. It is platform-neutral and belongs to the runtime; the +handlers modules re-export from here for their Telegram callers. + +Stdlib-only on purpose: everything that dispatches on a tool name (the danger +list, the risk gate, the confirmation summary) imports this module, so it must +never grow an import edge back into the runtime or the handlers. +""" + +from __future__ import annotations + +import json +from typing import Any + +# Tools that require user confirmation before execution +DANGEROUS_TOOLS = { + "place_order", + "manage_gateway_swaps", # execute action + "manage_gateway_clmm", # open/close position +} + +# Tools that are always blocked (RBAC bypass prevention) +BLOCKED_TOOLS: set[str] = set() + +# Actions within manage_executors that require confirmation +DANGEROUS_EXECUTOR_ACTIONS = {"create", "stop"} + +# Actions within manage_bots that deploy/mutate a live bot (status/logs/get_config +# are read-only and excluded). manage_controllers itself is excluded entirely — it +# only writes controller templates/saved configs, never a running bot (see its own +# tool docstring: "Does NOT affect running bots"). +DANGEROUS_BOT_ACTIONS = { + "deploy", + "stop_bot", + "stop_controllers", + "start_controllers", + "update_config", +} + +# Actions within manage_gateway_swaps that require confirmation +DANGEROUS_SWAP_ACTIONS = {"execute"} + +# Actions within manage_gateway_clmm that require confirmation +DANGEROUS_CLMM_ACTIONS = {"open_position", "close_position"} + + +def tool_call_name(tool_call: dict[str, Any]) -> str: + """The bare tool name, with any MCP prefix stripped. + + ACP names a tool by its wire name (``mcp__mcp-hummingbot__manage_bots``) + where the local surfaces use the bare one, so everything that dispatches on + a tool name — the danger list, the risk gate, the confirmation summary — + has to normalize identically or they disagree about the same call. + """ + raw_name = tool_call.get("tool", "") or tool_call.get("title", "") + return raw_name.rsplit("__", 1)[-1] if "__" in raw_name else raw_name + + +def tool_call_input(tool_call: dict[str, Any]) -> dict[str, Any] | None: + """A tool call's arguments as a mapping, or ``None`` when unreadable. + + ``None`` means "I cannot tell what this call does", and every caller has to + fail closed on it: an action-gated tool whose action can't be read is + treated as dangerous rather than waved through (SEC-093). Callers must not + reach for other spellings of the arguments — the ACP wire's ``rawInput`` is + translated once, at the boundary in :func:`condor.acp.client.normalize_tool_call`. + + A JSON string is parsed: OpenAI-compatible providers deliver tool arguments + that way. + """ + args = tool_call.get("input") + if isinstance(args, dict): + return args + if isinstance(args, str) and args.strip(): + try: + parsed = json.loads(args) + except json.JSONDecodeError: + return None + return parsed if isinstance(parsed, dict) else None + return None + + +def _has_dangerous_action( + tool_call: dict[str, Any], dangerous_actions: set[str] +) -> bool: + """Whether an action-gated tool call selects one of its dangerous actions. + + Fails closed (SEC-093): unreadable arguments, or a missing/non-string + ``action``, count as dangerous. ``action`` is required by every one of + these tools, so an unreadable one is never a benign read — it is a call we + failed to understand, and those belong in front of a human. + """ + input_data = tool_call_input(tool_call) + if input_data is None: + return True + action = input_data.get("action") + if not isinstance(action, str) or not action: + return True + return action in dangerous_actions + + +def is_dangerous_tool_call(tool_call: dict[str, Any]) -> bool: + """Check if a tool call requires user confirmation.""" + tool_name = tool_call_name(tool_call) + + # Direct dangerous tools + if tool_name in DANGEROUS_TOOLS: + # For manage_gateway_swaps, only "execute" action is dangerous + if tool_name == "manage_gateway_swaps": + return _has_dangerous_action(tool_call, DANGEROUS_SWAP_ACTIONS) + + # For manage_gateway_clmm, only open/close are dangerous + if tool_name == "manage_gateway_clmm": + return _has_dangerous_action(tool_call, DANGEROUS_CLMM_ACTIONS) + + return True + + # manage_executors with create/stop actions + if tool_name == "manage_executors": + return _has_dangerous_action(tool_call, DANGEROUS_EXECUTOR_ACTIONS) + + # manage_bots with deploy/stop/update actions (a bot deploy places real + # capital via a controller, a different path than manage_executors) + if tool_name == "manage_bots": + return _has_dangerous_action(tool_call, DANGEROUS_BOT_ACTIONS) + + return False + + +def format_tool_summary(tool_call: dict[str, Any]) -> str: + """Format a tool call into a human-readable summary for the confirmation message.""" + # The bare name, or an ACP call (``mcp__mcp-hummingbot__manage_bots``) would + # match none of the branches below and be approved as an opaque string. + tool_name = tool_call_name(tool_call) or "Unknown" + input_data = tool_call_input(tool_call) + if input_data is None: + # The gate sends unreadable calls here on purpose (SEC-093). Say so, + # rather than rendering a row of "?" that looks like a parsed call. + return f"{tool_name} (arguments could not be read)" + + if tool_name == "place_order": + side = input_data.get("trade_type", "?") + pair = input_data.get("trading_pair", "?") + amount = input_data.get("amount", "?") + order_type = input_data.get("order_type", "MARKET") + price = input_data.get("price", "") + connector = input_data.get("connector_name", "?") + summary = f"{side} {amount} {pair} ({order_type})" + if price: + summary += f" @ {price}" + summary += f" on {connector}" + return summary + + if tool_name == "manage_executors": + action = input_data.get("action", "?") + exec_type = input_data.get("executor_type", "") + exec_id = input_data.get("executor_id", "") + if action == "create" and exec_type: + config = input_data.get("executor_config", {}) + pair = config.get("trading_pair", "?") + return f"Create {exec_type} on {pair}" + if action == "stop" and exec_id: + return f"Stop executor {exec_id[:12]}..." + return f"Executor: {action}" + + if tool_name == "manage_bots": + action = input_data.get("action", "?") + bot_name = input_data.get("bot_name", "?") + if action == "deploy": + controllers = input_data.get("controllers_config", []) + return f"Deploy bot '{bot_name}' with controllers {controllers}" + if action == "update_config": + config_name = input_data.get("config_name", "?") + return f"Update config '{config_name}' on bot '{bot_name}'" + return f"Bot '{bot_name}': {action}" + + if tool_name == "manage_gateway_swaps": + action = input_data.get("action", "?") + pair = input_data.get("trading_pair", "?") + side = input_data.get("side", "?") + amount = input_data.get("amount", "?") + return f"Swap {side} {amount} {pair}" + + if tool_name == "manage_gateway_clmm": + action = input_data.get("action", "?") + if action == "open_position": + return "Open LP position" + if action == "close_position": + return "Close LP position" + return f"CLMM: {action}" + + # Generic fallback + return tool_name diff --git a/condor/runtime/llm_client.py b/condor/runtime/llm_client.py new file mode 100644 index 000000000..22f285cbc --- /dev/null +++ b/condor/runtime/llm_client.py @@ -0,0 +1,104 @@ +"""One factory for the PydanticAI-vs-ACP client split (ARCH-192). + +Every surface that talks to a model — chat sessions, consult, the trading-agent +engine — needs the same decision: a pydantic-ai key (ollama/lmstudio/openai/ +custom/...) gets an in-process :class:`PydanticAIClient`, anything else gets an +:class:`ACPClient` subprocess. That branch used to live in three drifted copies +(``runtime/sessions.py``, ``agents/consult.py``, ``agents/engine.py``); it now +lives here, and each surface passes only its own specifics: + +- chat sessions: ``extra_env`` (CONDOR_* ids), the bound-Agent ``system_prompt``, + ``strict_custom_endpoint=True`` (loud, actionable error for a missing saved + endpoint) and the LM Studio pref as ``default_base_url``. +- consult: nothing extra — it healthchecks the backend *before* calling this. +- engine: the run config's ``model_base_url`` as ``base_url_override``. + +The client classes are read as module attributes at call time so tests keep +patching the defining modules (``condor.acp.client.ACPClient`` / +``condor.acp.pydantic_ai_client.PydanticAIClient``) and intercept every surface +at once. +""" + +from __future__ import annotations + +import os +from typing import Any + +from condor.acp import client as acp_client +from condor.acp import pydantic_ai_client as pydantic_ai +from condor.preferences import resolve_custom_endpoint +from condor.runtime.toolsets import get_project_dir + + +def build_llm_client( + agent_key: str, + *, + mcp_servers: list[dict[str, Any]] | None = None, + permission_callback: acp_client.PermissionCallback | None = None, + allowed_tools: list[str] | None = None, + extra_env: dict[str, str] | None = None, + system_prompt: str = "", + user_data: dict | None = None, + user_id: int | None = None, + base_url_override: str | None = None, + default_base_url: str | None = None, + tool_filter_mode: str | None = None, + strict_custom_endpoint: bool = False, +) -> acp_client.ACPClient | pydantic_ai.PydanticAIClient: + """Build (but do not start) the right client for ``agent_key``. + + Custom-endpoint keys (``custom@:``) resolve through + :func:`condor.preferences.resolve_custom_endpoint` — pass ``user_data`` + when you have it (Telegram) or ``user_id`` when you don't (web, MCP, + background runs). ``strict_custom_endpoint=True`` keeps the chat path's + guided RuntimeError for a key naming an unsaved endpoint; the lenient + default falls back to the ``CUSTOM_LLM_*`` env vars. + + Base-URL precedence: ``base_url_override`` (an explicit config value, e.g. + the engine's ``model_base_url``) beats the resolved custom endpoint, which + beats ``default_base_url`` (a generic preference, e.g. the saved LM Studio + URL — deliberately last so it cannot shadow a named custom endpoint). + + ``tool_filter_mode`` resolves as: explicit value (a user/config pref) > + ``PYDANTIC_AI_TOOL_FILTER`` env > ``None`` (auto-detect by model size). + + ``extra_env``, ``system_prompt`` and ``allowed_tools`` are forwarded to + whichever client understands them: the ACP subprocess takes the env and the + system prompt but cannot enforce an allowlist; PydanticAI enforces the + allowlist (and takes the env for its MCP subprocesses). + """ + if pydantic_ai.is_pydantic_ai_model(agent_key): + custom_url, api_key = resolve_custom_endpoint( + agent_key, + user_data=user_data, + user_id=user_id, + strict=strict_custom_endpoint, + ) + return pydantic_ai.PydanticAIClient( + model=agent_key, + mcp_servers=mcp_servers, + permission_callback=permission_callback, + extra_env=extra_env, + base_url=base_url_override or custom_url or default_base_url or None, + api_key=api_key, + tool_filter_mode=( + tool_filter_mode or os.environ.get("PYDANTIC_AI_TOOL_FILTER") or None + ), + allowed_tools=allowed_tools, + ) + + # ACP subprocess models: claude-code, gemini, codex. A Claude model can be + # pinned via a suffix ("claude-acp:opus"); ACPClient selects it over the + # protocol (session/set_model) after handshake — the bridge ignores + # ANTHROPIC_MODEL. + command, model_env, model_pref = acp_client.resolve_acp(agent_key) + merged_env = {**(extra_env or {}), **model_env} + return acp_client.ACPClient( + command=command, + working_dir=get_project_dir(), + mcp_servers=mcp_servers, + permission_callback=permission_callback, + extra_env=merged_env or None, + model=model_pref or None, + system_prompt=system_prompt, + ) diff --git a/condor/runtime/sessions.py b/condor/runtime/sessions.py index d0a92d926..78d52b778 100644 --- a/condor/runtime/sessions.py +++ b/condor/runtime/sessions.py @@ -11,28 +11,21 @@ import asyncio import logging +from contextlib import asynccontextmanager from dataclasses import dataclass, field from datetime import datetime, timezone from telegram import Bot -from condor.acp import ACPClient, PermissionCallback, PromptDone, resolve_acp -from condor.acp.pydantic_ai_client import ( - PydanticAIClient, - is_pydantic_ai_model, - model_prefix, -) +from condor.acp import ACPClient, PermissionCallback, PromptDone +from condor.acp.pydantic_ai_client import PydanticAIClient from condor.agents.agent import identity_header as agent_identity_header from condor.runtime import binding, conversations from condor.runtime.confirmations import get_registry as get_confirmation_registry +from condor.runtime.context import build_initial_context, platform_formatting from condor.runtime.keys import SessionKey from condor.runtime.models import SessionInfo, SessionSpec from condor.runtime.timeouts import TIMEOUTS -from handlers.agents._shared import ( - build_initial_context, - get_project_dir, - platform_formatting, -) log = logging.getLogger(__name__) @@ -51,6 +44,44 @@ # Not persisted -- subprocesses can't survive restarts. _sessions: dict[str, "AgentSession"] = {} +# Creation serialization (CORR-187). get_or_create_session is a check-then- +# create that spans many awaits (subprocess spawn, eager context prompt), so +# without a lock two concurrent calls for the same key both spawn a full +# client and the second registration overwrites the first, whose process tree +# is never stopped -- teardown only walks the registry. Locks are per session +# key (creations under different keys stay concurrent) and refcounted so the +# dict does not grow with every key ever seen. +_creation_locks: dict[str, asyncio.Lock] = {} +_creation_lock_refs: dict[str, int] = {} + +# Session keys currently being created, per user. A creation in flight is not +# in ``_sessions`` yet but will register one session, so the budget check has +# to count it or N concurrent creates of distinct keys all pass the cap. +_pending_creates: dict[int, set[str]] = {} + + +@asynccontextmanager +async def _creation_lock(name: str): + """Hold the named creation lock; drop it when the last holder leaves. + + Refcounted rather than popped on release: popping while another task still + waits on the lock object would hand a *fresh* lock to the next caller and + reopen the race between the waiter and the newcomer. + """ + lock = _creation_locks.setdefault(name, asyncio.Lock()) + _creation_lock_refs[name] = _creation_lock_refs.get(name, 0) + 1 + try: + async with lock: + yield + finally: + remaining = _creation_lock_refs[name] - 1 + if remaining: + _creation_lock_refs[name] = remaining + else: + del _creation_lock_refs[name] + _creation_locks.pop(name, None) + + # Health monitor state _health_task: asyncio.Task | None = None _health_bot: Bot | None = None @@ -244,7 +275,11 @@ async def _enforce_session_budget(user_id: int) -> None: while True: mine = [s for s in _sessions.values() if s.user_id == user_id] - if len(mine) < MAX_SESSIONS_PER_USER: + # Creations in flight for this user (reserved under the per-user + # creation lock, registered only later) hold budget too, or N + # concurrent creates of distinct keys would all pass the count. + in_flight = len(_pending_creates.get(user_id, set()) - set(_sessions)) + if len(mine) + in_flight < MAX_SESSIONS_PER_USER: return idle = [s for s in mine if not s.is_busy] @@ -357,6 +392,25 @@ async def get_or_create_session( """ key = SessionKey.parse(spec.key) raw_key = str(key) + + # One creation at a time per key (CORR-187): the second concurrent caller + # waits here and then finds the first caller's session in the registry + # instead of spawning a duplicate whose twin would leak until shutdown. + # Locks are per key, so creations under different keys stay concurrent. + async with _creation_lock(raw_key): + return await _get_or_create_session_locked( + spec, key, raw_key, permission_callback, user_data + ) + + +async def _get_or_create_session_locked( + spec: SessionSpec, + key: SessionKey, + raw_key: str, + permission_callback: PermissionCallback | None, + user_data: dict | None, +) -> AgentSession: + """Body of :func:`get_or_create_session`; runs under the per-key lock.""" session = _sessions.get(raw_key) # Reuse existing session only if the same brain is still on the other end: @@ -384,11 +438,47 @@ async def get_or_create_session( # Destroy old session if exists if session: await _destroy_session_internal(key) - elif spec.user_id: - # Only a genuinely new key counts against the budget — replacing the - # session behind an existing key is a swap, not an extra subprocess. - await _enforce_session_budget(spec.user_id) + if spec.user_id: + if session is None: + # Only a genuinely new key counts against the budget — replacing + # the session behind an existing key is a swap, not an extra + # subprocess. + await _enforce_session_budget(spec.user_id) + # Reserve this key's budget slot for the whole spawn. The reservation + # must follow the check with no await in between — the event loop + # makes the pair atomic, so two creates can never both pass the count + # before either reserves (and _enforce_session_budget recounts after + # every await it does make). The swap path reserves too: its old + # session is already out of the registry, so a concurrent create + # would otherwise undercount during the respawn window. + _pending_creates.setdefault(spec.user_id, set()).add(raw_key) + try: + return await _spawn_session( + spec, key, raw_key, permission_callback, user_data + ) + finally: + reserved = _pending_creates.get(spec.user_id) + if reserved is not None: + reserved.discard(raw_key) + if not reserved: + del _pending_creates[spec.user_id] + + return await _spawn_session(spec, key, raw_key, permission_callback, user_data) + + +async def _spawn_session( + spec: SessionSpec, + key: SessionKey, + raw_key: str, + permission_callback: PermissionCallback | None, + user_data: dict | None, +) -> AgentSession: + """Spawn, contextualize, and register a new session for ``raw_key``. + + Runs under the per-key creation lock, with the key's budget slot already + reserved when the spec names a user. + """ # MCP subprocess env expects a numeric chat id; surfaces without a chat # (web, mcp) fall back to the user id. The same pair goes down on argv via # binding.resolve, which is what the subprocess actually reads first — so @@ -413,84 +503,40 @@ async def get_or_create_session( mcp_servers = bound.mcp_servers agent_key = bound.agent_key or spec.agent_key - # Check if agent_key requires PydanticAI client (ollama, lmstudio, openai, etc.) - use_pydantic_ai = is_pydantic_ai_model(agent_key) - - if use_pydantic_ai: - # For Pydantic AI models: auto-detect or use configured filter mode - import os - - from condor.preferences import get_agent_prefs - - # Priority: user preference > env variable > auto-detect (None) - agent_prefs = get_agent_prefs(user_data) if user_data else {} - tool_filter_mode = ( - agent_prefs.get("tool_filter_mode") - or os.environ.get("PYDANTIC_AI_TOOL_FILTER") - or None # None triggers auto-detection based on model size - ) - - base_url = ( + # One factory decides PydanticAI vs ACP for every surface (ARCH-192). + # Chat-session specifics: the CONDOR_* env pair, the bound-Agent identity + # header (system level — the opening context below is a user turn and loses + # to the host's own system prompt, FEAT-025), the saved LM Studio pref as + # the *default* base URL (a named custom endpoint still wins), and + # strict_custom_endpoint=True so a key naming an unsaved endpoint fails + # loudly with the guided RuntimeError instead of dying deep in httpx. + import os + + from condor.preferences import get_agent_prefs + from condor.runtime.llm_client import build_llm_client + + agent_prefs = get_agent_prefs(user_data) if user_data else {} + client = build_llm_client( + agent_key, + mcp_servers=mcp_servers, + permission_callback=permission_callback, + # A bound Agent's allowlist is enforced here exactly as it is on + # consult and loop, so an Agent has the same reach in every mode. + allowed_tools=bound.tools or None, + extra_env=extra_env, + system_prompt=( + agent_identity_header(bound.agent_slug, bound.label) + if bound.is_agent + else "" + ), + user_data=user_data, + user_id=spec.user_id, + default_base_url=( agent_prefs.get("base_url") or os.environ.get("LMSTUDIO_BASE_URL") or None - ) - - api_key = None - if model_prefix(agent_key) == "custom": - # Custom OpenAI-compatible provider. The agent key names one of the - # user's saved endpoints ("custom@venice:..."); those live in the - # shared preference store so Telegram and the web dashboard resolve - # them identically. CUSTOM_LLM_* env vars cover headless deploys. - from condor.preferences import find_custom_provider, parse_custom_agent_key - - provider_name, _ = parse_custom_agent_key(agent_key) - provider = ( - find_custom_provider(user_data, provider_name) if user_data else None - ) - if provider is None and provider_name: - raise RuntimeError( - f"No saved endpoint named '{provider_name}'. Add it via " - "/agent → Change LLM → Custom endpoint, or Settings → " - "AI Providers on the web dashboard." - ) - provider = provider or {} - base_url = provider.get("base_url") or os.environ.get("CUSTOM_LLM_BASE_URL") - api_key = provider.get("api_key") or os.environ.get("CUSTOM_LLM_API_KEY") - - client = PydanticAIClient( - model=agent_key, - mcp_servers=mcp_servers, - permission_callback=permission_callback, - extra_env=extra_env, - tool_filter_mode=tool_filter_mode, # Auto-detects if None - base_url=base_url, - api_key=api_key, - # A bound Agent's allowlist is enforced here exactly as it is on - # consult and loop, so an Agent has the same reach in every mode. - allowed_tools=bound.tools or None, - ) - else: - # For ACP subprocess models: claude-code, gemini, codex. - # A Claude model can be pinned via a suffix, e.g. "claude-acp:opus" / - # "claude-acp:sonnet"; ACPClient selects it via session/set_model after - # handshake (the bridge ignores ANTHROPIC_MODEL). Bare key = agent default. - command, model_env, model_pref = resolve_acp(agent_key) - client = ACPClient( - command=command, - working_dir=get_project_dir(), - mcp_servers=mcp_servers, - permission_callback=permission_callback, - extra_env={**extra_env, **model_env}, - model=model_pref, - # Who this brain IS, at system level. The opening context below is a - # user turn: it is read once and then loses to the host's own system - # prompt, which is why a bound Agent kept answering as Condor - # (FEAT-025). This is the one channel that outranks it. - system_prompt=( - agent_identity_header(bound.agent_slug, bound.label) - if bound.is_agent - else "" - ), - ) + ), + tool_filter_mode=agent_prefs.get("tool_filter_mode"), + strict_custom_endpoint=True, + ) await client.start() diff --git a/condor/runtime/toolsets.py b/condor/runtime/toolsets.py new file mode 100644 index 000000000..f52a9a44d --- /dev/null +++ b/condor/runtime/toolsets.py @@ -0,0 +1,240 @@ +"""MCP toolset builders for agent sessions (ARCH-190). + +Builds the per-session MCP server configs (condor + hummingbot subprocesses) +that every surface — Telegram, web dashboard, consult/delegate, the tick +engine — hands to its client. Platform-neutral runtime foundation: this used +to live in ``handlers/agents/_shared.py``, which the runtime could only reach +via lazy imports; ``handlers`` now re-exports from here instead. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import Any + +from condor.acp.client import bot_process_marker + +log = logging.getLogger(__name__) + + +def get_project_dir() -> str: + """Get the condor project root directory (where .mcp.json lives). + + The ACP agent auto-discovers stdio MCP servers from .mcp.json in the cwd, + so we just need to point it at the project root. + """ + return str(Path(__file__).parent.parent.parent) + + +def _bot_token() -> str: + """This bot's Telegram token, read at call time. + + Goes through ``utils.config`` for its ``load_dotenv()`` side effect, so the + marker built here is derived from the same token ``main.py`` hands the + reaper — a mismatch would silently strand orphaned subprocess trees. + """ + import utils.config # noqa: F401 (imported for load_dotenv()) + + return os.environ.get("TELEGRAM_TOKEN", "") + + +def _bot_id_args() -> list[str]: + """``--bot-id `` for an MCP subprocess, or nothing without a token. + + The digest is what the startup reaper (``reap_stale_acp_trees``) seeds on to + find subprocess trees orphaned by a previous crash. It replaced the raw + token, which used to sit here in clear text (SEC-095). Both MCP servers + parse their argv with ``parse_known_args``, so the flag is inert to them — + it exists purely to be visible in ``ps``. + """ + marker = bot_process_marker(_bot_token()) + return ["--bot-id", marker] if marker else [] + + +def _env_entries(**values: Any) -> list[dict[str, str]]: + """ACP ``env`` entries (``{"name", "value"}``), skipping empty values. + + This is the channel every secret an MCP subprocess needs must travel on: the + ACP bridge and the pydantic-ai stdio client both overlay these onto the + child's inherited environment, which — unlike argv — no other local user can + read out of ``ps``. + + Values are coerced to ``str``: YAML loads an unquoted numeric credential as + int (e.g. ``password: 123``), and an int in the environment mapping breaks + subprocess spawning — for pydantic-ai backends (lmstudio:/ollama:/ + openrouter:) it surfaces as a StdioServerParameters validation error. + """ + entries = ( + (name, str(value)) for name, value in values.items() if value is not None + ) + return [{"name": name, "value": value} for name, value in entries if value] + + +def _condor_mcp_args( + chat_id: int | str, + user_id: int, + agent_slug: str | None = None, + server_name: str | None = None, + delegate_worker: bool = False, +) -> list[str]: + """Build CLI args for the condor MCP subprocess. + + ``delegate_worker`` marks a *background Condor worker* — the detached session + ``delegate`` starts (FEAT-032). The chat and that worker share one agent + record, so the flag is what tells the subprocess which seat it is sitting in. + + The bot token travels in the server's ``env``, never here: argv is + world-readable via ``ps`` (SEC-095). What argv carries instead is + ``--bot-id``, the token's non-secret digest, which is how the startup reaper + recognizes our own leaked subprocess trees. + """ + # MCP server expects int chat_id. For web sessions (string keys like "web_42"), + # use user_id instead — in Telegram DMs, chat_id == user_id anyway. + effective_chat_id = chat_id if isinstance(chat_id, int) else user_id + args = [ + "--chat-id", + str(effective_chat_id), + "--user-id", + str(user_id), + ] + args.extend(_bot_id_args()) + if agent_slug: + args.extend(["--agent-slug", str(agent_slug)]) + if server_name: + args.extend(["--server-name", str(server_name)]) + if delegate_worker: + args.append("--delegate-worker") + return args + + +def _hummingbot_mcp_args(server: dict[str, Any], server_name: str) -> list[str]: + """Build CLI args for the hummingbot MCP subprocess. + + Only non-secret coordinates travel here — the API username/password go in + the server's ``env`` instead, since argv is world-readable via ``ps`` + (SEC-095). Every element is a string: YAML loads unquoted numerics as int + (e.g. ``port: 8000``), and pydantic-ai's StdioServerParameters rejects + non-string args when starting LM Studio / other local-model sessions. + """ + api_url = f"http://{server['host']}:{server['port']}" + return [ + "run", + "python", + "-m", + "mcp_servers.hummingbot_api", + "--url", + api_url, + "--server-name", + str(server_name), + ] + _bot_id_args() + + +def build_mcp_servers_for_session( + user_id: int, + chat_id: int | str, + user_data: dict | None = None, + server_name: str | None = None, + agent_slug: str | None = None, + delegate_worker: bool = False, +) -> list[dict[str, Any]]: + """Build dynamic MCP server configs for an agent session. + + Returns ACP-format mcpServers that override the static .mcp.json entries by + name. Always includes the condor MCP server; hummingbot is added when a + valid server can be resolved for the user. + + ``server_name`` pins the run to that Condor server (an Agent with + ``server_required``); when omitted, the chat's ambient server is resolved + from the user's preferences, then from the first accessible server. Each + candidate must exist *and* be reachable by ``user_id`` — a name this user + has no grant on resolves to the next candidate, never to its credentials. + + ``agent_slug`` scopes the condor MCP tools' memory/skills to that Agent's + own stores (``agents/{slug}/``). Without it the tools target the chat + condor's stores — correct for chat sessions, wrong for an Agent run: a + serverless consult/tick would silently read and write the CHAT's memory + and skills instead of the Agent's own (e.g. an agent unable to find its + inherited ``routine_cookbook`` skill). + + ``delegate_worker`` marks a background Condor delegation (FEAT-032) so the + subprocess picks up the worker framing and refuses to delegate again. + """ + from config_manager import get_config_manager, get_effective_server + + cm = get_config_manager() + + # Resolve which hummingbot server to use (explicit override > user + # preferences). Every candidate is held to existence *and* reach, because + # the name that comes out of here decides whose API credentials go into the + # subprocess env below. ``chat_id`` is not a principal — ``chat_defaults`` + # is a global map keyed by chat, so a caller who can name someone else's + # chat could name their server too (SEC-178, the SEC-164 shape). The subject + # is always ``user_id``, the authenticated owner of the run; the same + # predicate guards TickEngine._resolve_server. + def usable(name: str | None) -> bool: + return bool(name and cm.get_server(name)) and cm.has_server_access( + user_id, name + ) + + def candidates(): + # Lazy on purpose: resolving the chat default writes back into + # ``user_data``, so it must not run when a pin already answered. + yield server_name + yield get_effective_server(chat_id, user_data) + accessible = cm.get_accessible_servers(user_id) + yield accessible[0] if accessible else None + + server_name = next((name for name in candidates() if usable(name)), None) + + # Condor MCP -- runs as stdio subprocess, tools work locally without TCP bridge + # Pass resolved server_name so start_agent uses the correct server + condor = { + "name": "condor", + "command": "uv", + "args": ["run", "python", "-m", "mcp_servers.condor"] + + _condor_mcp_args( + chat_id, + user_id, + agent_slug, + server_name=server_name, + delegate_worker=delegate_worker, + ), + "env": _env_entries(TELEGRAM_BOT_TOKEN=_bot_token()), + } + + if not server_name: + log.warning( + "No accessible server for user %s (chat %s) — " + "agent will start without mcp-hummingbot", + user_id, + chat_id, + ) + return [condor] + + server = cm.get_server(server_name) + if not server: + log.warning( + "Server '%s' resolved for user %s but not found in servers config — " + "agent will start without mcp-hummingbot", + server_name, + user_id, + ) + return [condor] + + # Credentials go in env, not argv: the API username/password used to sit on + # the command line, where any local `ps` recovered them (SEC-095). The + # non-secret coordinates (url, server name) stay on argv, where they make a + # running subprocess identifiable. + mcp_hummingbot = { + "name": "mcp-hummingbot", + "command": "uv", + "args": _hummingbot_mcp_args(server, server_name), + "env": _env_entries( + HUMMINGBOT_API_USERNAME=server["username"], + HUMMINGBOT_API_PASSWORD=server["password"], + ), + } + + return [mcp_hummingbot, condor] diff --git a/condor/server_data_service.py b/condor/server_data_service.py index 318f50fa8..a6dce8894 100644 --- a/condor/server_data_service.py +++ b/condor/server_data_service.py @@ -896,10 +896,10 @@ def register_default_fetches() -> None: sds.register_fetch(ServerDataType.PORTFOLIO, fetch_portfolio) sds.register_fetch(ServerDataType.PORTFOLIO_HISTORY, fetch_portfolio_history) - sds.register_fetch(ServerDataType.PRICES, fetch_current_price) # strict=True on the cached reads: a failed call must reach # _do_fetch_and_cache's except branch (record_error, keep the last good # value, back off) instead of being cached as "nothing here". + sds.register_fetch(ServerDataType.PRICES, partial(fetch_current_price, strict=True)) sds.register_fetch(ServerDataType.POSITIONS, partial(fetch_positions, strict=True)) sds.register_fetch( ServerDataType.ACTIVE_ORDERS, partial(fetch_active_orders, strict=True) @@ -917,8 +917,10 @@ def register_default_fetches() -> None: sds.register_fetch(ServerDataType.BOT_RUNS, fetch_bot_runs) sds.register_fetch(ServerDataType.CANDLE_CONNECTORS, fetch_candle_connectors) sds.register_fetch(ServerDataType.SERVER_STATUS, fetch_server_status) - sds.register_fetch(ServerDataType.TICKERS, fetch_tickers) - sds.register_fetch(ServerDataType.TICKER_POOL, fetch_ticker_pool) + sds.register_fetch(ServerDataType.TICKERS, partial(fetch_tickers, strict=True)) + sds.register_fetch( + ServerDataType.TICKER_POOL, partial(fetch_ticker_pool, strict=True) + ) logger.info( "ServerDataService: registered fetch functions for %d data types", diff --git a/condor/setup_llm.py b/condor/setup_llm.py index fac60277f..1fa7e000f 100644 --- a/condor/setup_llm.py +++ b/condor/setup_llm.py @@ -8,7 +8,7 @@ prints the same report and changes nothing. The wizard owns none of the detection: it renders -:mod:`handlers.agents.readiness` over :data:`handlers.agents._shared.AGENT_OPTIONS`, +:mod:`condor.llm.readiness` over :data:`condor.llm.options.AGENT_OPTIONS`, so a provider added to either shows up here for free and can never disagree with what the ``get_available_models`` MCP tool reports. @@ -29,14 +29,14 @@ import httpx from condor.acp.pydantic_ai_client import DEFAULT_BASE_URLS -from handlers.agents import readiness -from handlers.agents._shared import ( +from condor.llm import readiness +from condor.llm.openrouter_models import fetch_models +from condor.llm.options import ( AGENT_OPTIONS, RECOMMENDED_AGENT, selectable_agent_options, ) -from handlers.agents.openrouter_models import fetch_models -from handlers.agents.readiness import MISSING, READY, UNVERIFIED, Readiness +from condor.llm.readiness import MISSING, READY, UNVERIFIED, Readiness REPO_ROOT = Path(__file__).resolve().parent.parent ENV_PATH = REPO_ROOT / ".env" diff --git a/condor/telemetry/__init__.py b/condor/telemetry/__init__.py index 577606190..8f7bc13a8 100644 --- a/condor/telemetry/__init__.py +++ b/condor/telemetry/__init__.py @@ -1,23 +1,24 @@ -"""Anonymous, opt-in usage telemetry (FEAT-023). +"""Anonymous usage telemetry (FEAT-023). -Condor is self-hosted, so the project sees nothing about how installs are used -unless an install chooses to tell it. This package is that channel — and, -because the same process holds exchange API keys, it is built to be auditable in -one sitting rather than to be clever. +Condor is self-hosted, so beyond knowing an install exists the project sees +nothing about how it is used unless the install chooses to tell it. This +package is that channel — and, because the same process holds exchange API +keys, it is built to be auditable in one sitting rather than to be clever. The four facts that define it: -- **Off by default.** A fresh clone has no consent recorded, which resolves to - level ``off``, at which :func:`emit` returns before doing anything at all. No - buffer, no file, no directory. -- **Opt-in, once, by the admin.** One inline-keyboard prompt on boot offers - full usage, install-count-only, or no. The answer is durable and reversible. +- **Every install is counted.** The floor is level ``ping``: the four adoption + events (``install``, ``heartbeat``, ``version_change``, ``shutdown``) and the + anonymous envelope, from the first boot, with no answer required. +- **Usage is opt-in, once, by the admin.** One inline-keyboard prompt on boot + offers full usage or install-count-only. The answer is durable and + reversible. - **Allowlisted.** :mod:`condor.telemetry.schema` declares every event and every property. Anything undeclared is dropped by construction, which is what makes the "never collected" list in ``PRIVACY.md`` a property of the code. -- **Nothing transmits without an endpoint.** No collector URL is compiled in. - Unset ``CONDOR_TELEMETRY_URL`` means the send path is inert and events only - ever reach a capped local file. +- **The environment is the only kill switch.** The collector address is fixed + in :mod:`condor.telemetry.outbox`; ``CONDOR_TELEMETRY=off`` is the one way to + reach level ``off``, at which nothing is ever recorded. Public surface — call sites should need nothing else:: @@ -61,10 +62,17 @@ def init(hosted: bool = True) -> str: ``hosted`` marks a process that owns a flush job, so events buffer in memory instead of going straight to a spool file. The MCP subprocess passes False. - Priming does two things and only two: it resolves the level once so + Priming does three things and only three: it resolves the level once so :func:`emit` never has to read the disk on a hot path, and — only if the - level is not ``off`` — it materializes the install's random ids. An install - that never opted in is left exactly as it was found. + level is not ``off`` — it materializes the install's random ids and counts + the install. With the ping floor all three happen on the very first boot; + only an install silenced by ``CONDOR_TELEMETRY=off`` is left exactly as it + was found. + + Counting here, rather than when the admin answers the prompt, is what makes + "every install is counted" true rather than aspirational: an install whose + admin ignores the prompt — and a local-mode install, which has no prompt to + ignore — is still an install. ``mark_install_reported`` keeps it to once. """ from condor.telemetry import consent, emitter @@ -72,6 +80,8 @@ def init(hosted: bool = True) -> str: effective = consent.refresh() if effective != consent.OFF: consent.ensure_identity() + if consent.mark_install_reported(): + emit("install") return effective diff --git a/condor/telemetry/consent.py b/condor/telemetry/consent.py index d59c84167..0ce169ed4 100644 --- a/condor/telemetry/consent.py +++ b/condor/telemetry/consent.py @@ -1,26 +1,25 @@ -"""Consent: the three-state machine that decides whether anything happens at all. +"""Consent: the machine that decides how much an install says. The install — not the individual user — is the unit of consent, because the admin owns the install. The states are ``unknown`` (the default on a fresh -clone), ``granted`` and ``denied``, stored in ``config.yml`` under ``telemetry`` -alongside the install's identity. +clone), ``granted`` and ``denied`` (legacy — older installs could refuse from +the prompt), stored in ``config.yml`` under ``telemetry`` alongside the +install's identity. Two rules matter more than the rest: -**Unknown means off.** Not "buffered until someone decides" — off. An install -whose admin has never answered the prompt collects nothing, holds nothing in -memory, and writes no spool file. The original design buffered pre-consent -events and simply refused to send them; recording data before permission is a -worse default than losing it, so this narrows that decision. Nothing is lost -that we had any right to. +**The floor is ``ping``.** Every install is counted. An install whose admin has +never answered the prompt emits the four adoption events (``install``, +``heartbeat``, ``version_change``, ``shutdown``) and nothing else; the prompt +decides one thing only — whether the ``usage`` events are added on top. There +is no "off" answer on the form. **The environment wins.** ``CONDOR_TELEMETRY`` in ``utils/config.py`` overrides -the stored answer in both directions, so a headless or containerized install is -never blocked on a Telegram prompt, and an operator can force ``off`` no matter -what is on disk. +the stored answer in both directions, and it is the only way to reach level +``off`` — the operator's kill switch for a headless or containerized install. -Reads never create ``config.yml``. On an install that has not been configured at -all, :func:`state` answers ``unknown`` from nothing. +Reads never create ``config.yml``; the identity ids are materialized by +:func:`condor.telemetry.init` on the first boot that can actually emit. """ from __future__ import annotations @@ -41,19 +40,14 @@ GRANTED = "granted" DENIED = "denied" -# Answer -> level, the three buttons of the admin prompt. -ANSWER_LEVELS = {"usage": USAGE, "ping": PING, "off": OFF} +# Answer -> level, the two buttons of the admin prompt. "off" is deliberately +# not an answer: install counting is the floor, and only the environment can +# silence it entirely. +ANSWER_LEVELS = {"usage": USAGE, "ping": PING} _cached_level: str | None = None -def _config_path() -> Path: - from config_manager import ConfigManager - - existing = ConfigManager._instance - return Path(existing.config_path if existing else "config.yml") - - def _cm(): """The ConfigManager, but only if reading it cannot create a config file.""" from config_manager import ConfigManager @@ -120,10 +114,12 @@ def _compute_level() -> str: env = _env_level() if env is not None: return env - if state() != GRANTED: - return OFF - stored = _section().get("level") - return stored if stored in LEVELS else USAGE + if state() == GRANTED: + stored = _section().get("level") + return stored if stored in (PING, USAGE) else USAGE + # Unanswered — and legacy denied — installs are still counted: ping is the + # floor, and only the environment can force `off`. + return PING def refresh() -> str: @@ -179,10 +175,13 @@ def install_secret() -> str: def grant(answer: str) -> str: - """Record a positive answer. ``answer`` is one of :data:`ANSWER_LEVELS`.""" - chosen = ANSWER_LEVELS.get(answer, USAGE) - if chosen == OFF: - return deny() + """Record the admin's answer. ``answer`` is one of :data:`ANSWER_LEVELS`. + + Anything unrecognized — including an "off" tap on a prompt left over from + an older version — lands on ``ping``, the floor, never silently on + ``usage``. + """ + chosen = ANSWER_LEVELS.get(answer, PING) _update( consent=GRANTED, level=chosen, @@ -192,43 +191,62 @@ def grant(answer: str) -> str: return chosen -def deny() -> str: - """Record a refusal and destroy whatever was already on disk. +def _purge_collected() -> None: + """Destroy everything recorded but not yet sent. - Denial deletes the spool and the outbox rather than merely ignoring them: - an answer of "no" should leave nothing behind to be sent by a later bug. + Withdrawing usage consent deletes the spool and the outbox rather than + merely ignoring them: a downgrade should leave nothing behind to be sent + by a later bug. """ - _update( - consent=DENIED, - level=OFF, - decided_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - ) from condor.telemetry import emitter, outbox emitter.discard_buffer() outbox.purge() - return OFF def set_level(new_level: str) -> str: - """Change the level of an install that has already consented.""" - if new_level not in LEVELS: + """Change the level of an install. ``off`` is not accepted here: the floor + is ``ping``, and only ``CONDOR_TELEMETRY=off`` reaches ``off``.""" + if new_level not in (PING, USAGE): return level() - if new_level == OFF: - return deny() + downgrading = new_level == PING and level() == USAGE _update(consent=GRANTED, level=new_level) ensure_identity() + if downgrading: + _purge_collected() return new_level +# ── Install counting ───────────────────────────────────────────────────── + + +def mark_install_reported() -> bool: + """True the first time this install is counted, False forever after. + + The ``install`` event carries no properties — it *is* the count — so it has + to be emitted exactly once per install, by whichever surface reaches it + first: boot, the Telegram prompt, or the dashboard card. + + It used to be emitted only from the Telegram consent callback, which meant + an install that never answered was never counted at all — and a local-mode + install, which has no prompt to answer, could not be counted even in + principle. ``ping`` is the floor, so counting needs no answer; this flag is + what keeps it from being counted twice. + """ + if _section().get("install_reported"): + return False + _update(install_reported=True) + return True + + # ── First-use tracking ─────────────────────────────────────────────────── def mark_feature_seen(feature: str) -> bool: """True the first time this install ever uses ``feature``, False after. - Backs the ``feature_first_use`` activation funnel. Only called when - telemetry is on, so an opted-out install never accumulates the list. + Backs the ``feature_first_use`` activation funnel. Only called at level + ``usage``, so an install that has not opted in never accumulates the list. """ section = _section() seen = list(section.get("features_seen") or []) diff --git a/condor/telemetry/context.py b/condor/telemetry/context.py index e5202e826..e12c4c360 100644 --- a/condor/telemetry/context.py +++ b/condor/telemetry/context.py @@ -95,9 +95,25 @@ def _llm_providers() -> list[str]: return found +def _mode() -> str: + """``telegram`` | ``local``. Which surface this install actually runs. + + One of two fixed names, never a token, a chat id or a host — it answers + "how many installs run Condor without Telegram at all?", which is the whole + point of local mode and was previously invisible. + """ + try: + from utils.config import CONDOR_MODE + + return CONDOR_MODE + except Exception: + return "unknown" + + def config_shape() -> dict: """Counts and capability flags. Every field is a bool, an int, or a fixed name.""" shape: dict = { + "mode": _mode(), "has_web": True, "has_gateway": False, "has_hb_api": False, diff --git a/condor/telemetry/emitter.py b/condor/telemetry/emitter.py index b740b5468..5010ab870 100644 --- a/condor/telemetry/emitter.py +++ b/condor/telemetry/emitter.py @@ -8,9 +8,11 @@ the fallback is silence. 2. **It never blocks and never does I/O** in the host process. It appends to a bounded ``deque``; the network is somebody else's job, 15 minutes later. -3. **It is off unless someone said yes.** The first gate is a cached level read, - and on an install that has not opted in that read returns ``off`` and the - function returns before it has looked at its own arguments. +3. **It says no more than the level allows.** The first gate is a cached level + read: ``CONDOR_TELEMETRY=off`` makes it return before it has looked at its + own arguments, and at the default ``ping`` floor only the four adoption + events pass the schema gate — everything else needs an explicit ``usage`` + opt-in. 4. **It cannot emit what the schema does not declare.** Properties are allowlisted, type-checked and truncated by :mod:`condor.telemetry.schema`. @@ -31,7 +33,6 @@ RING_MAX = 2000 RATE_PER_MIN = 60 -FLUSH_THRESHOLD = 200 _buffer: deque[dict] = deque(maxlen=RING_MAX) _dropped = 0 @@ -48,10 +49,6 @@ def set_hosted(value: bool) -> None: _hosted = value -def is_hosted() -> bool: - return _hosted - - def _take_token(name: str) -> bool: """Token bucket, :data:`RATE_PER_MIN` per event name per minute.""" now = time.monotonic() @@ -118,10 +115,6 @@ def dropped() -> int: return _dropped -def should_flush() -> bool: - return len(_buffer) >= FLUSH_THRESHOLD - - def drain() -> tuple[list[dict], int]: """Take the ring and the dropped count together, and reset both.""" global _dropped @@ -132,7 +125,8 @@ def drain() -> tuple[list[dict], int]: def discard_buffer() -> None: - """Throw the ring away without sending it. Used when consent is denied.""" + """Throw the ring away without sending it. Used when usage consent is + withdrawn.""" global _dropped _buffer.clear() _dropped = 0 @@ -142,14 +136,16 @@ def discard_buffer() -> None: async def flush(reason: str = "job") -> int: """Try to deliver everything pending. Returns the number of events sent. - Returns 0 without reading a file when consent is not granted, and 0 without - touching the network when no endpoint is configured — the shipped state, in - which events simply accumulate in the capped outbox. + Returns 0 without reading a file when the operator has forced ``off``, and + 0 without touching the network when no endpoint is configured — the shipped + state, in which events simply accumulate in the capped outbox. An + unanswered install flushes too: the ping floor is only worth anything if + the adoption events actually leave. """ try: from condor.telemetry import consent, context, outbox - if consent.state() != consent.GRANTED or consent.level() == consent.OFF: + if consent.level() == consent.OFF: return 0 events, dropped_count = drain() @@ -159,8 +155,6 @@ async def flush(reason: str = "job") -> int: return 0 if not outbox.endpoint(): - # Nowhere to go yet (the collector is FEAT-024). Park them; the cap - # in outbox.py is what keeps this honest on a long-running install. outbox.stash(events) return 0 diff --git a/condor/telemetry/outbox.py b/condor/telemetry/outbox.py index a11b74871..52eb6d326 100644 --- a/condor/telemetry/outbox.py +++ b/condor/telemetry/outbox.py @@ -15,11 +15,10 @@ so an install that never reaches a collector accumulates a bounded file and then quietly drops the excess. That is the intended behaviour, not a bug. -**Nothing is transmitted unless an operator sets ``CONDOR_TELEMETRY_URL``.** No -collector address is compiled into this repository. With the variable unset — -which is the shipped state — :func:`post` returns ``False`` without importing a -network client, and every event's whole life is a local file that -:func:`purge` can delete. +The collector address is :data:`COLLECTOR_URL`, compiled in and not +configurable. Whether anything is sent to it is decided entirely by consent: at +level ``off`` — the default — no event is ever created, so :func:`post` is never +reached, and :func:`purge` deletes whatever a withdrawn opt-in left behind. """ from __future__ import annotations @@ -43,6 +42,7 @@ # outbox could never drain. Keep this at or below the smallest cap we target. MAX_BATCH_EVENTS = 500 POST_TIMEOUT_S = 10 +COLLECTOR_URL = "https://telemetry.hummingbot.org/v1/events" def root() -> Path: @@ -206,16 +206,13 @@ def batches(events: list[dict]) -> list[list[dict]]: return out -def endpoint() -> str | None: - """The collector URL, or None — which is the shipped default.""" - from utils.config import CONDOR_TELEMETRY_URL - - return CONDOR_TELEMETRY_URL +def endpoint() -> str: + """The collector URL.""" + return COLLECTOR_URL async def post(envelope: dict) -> bool: - """Deliver one envelope. Returns False — without touching the network — when - no endpoint is configured, which is the state this repository ships in.""" + """Deliver one envelope.""" url = endpoint() if not url: return False diff --git a/condor/telemetry/prompt.py b/condor/telemetry/prompt.py index dd3ee0957..c37529204 100644 --- a/condor/telemetry/prompt.py +++ b/condor/telemetry/prompt.py @@ -1,10 +1,16 @@ """The one-tap consent prompt, and the callback that answers it. -Opt-in only works if asking is cheap, so this is a single message with three +Opt-in only works if asking is cheap, so this is a single message with two buttons next to the "Condor is online" notification the admin already gets. It is sent at most once per version, the intent is written to disk *before* the message goes out (a crash loop must not re-ask forever), and until it is -answered the install emits nothing at all. +answered the install stays at the ``ping`` floor — counted, nothing more. + +Telegram is not the only surface that asks. A local-mode install has no bot to +message, so the dashboard asks instead — ``GET /api/v1/settings/telemetry`` +serves :data:`DISCLOSURE` to the consent card. Both surfaces render the same +copy from the same constant: a privacy claim written down twice is a privacy +claim that will eventually disagree with itself. """ from __future__ import annotations @@ -15,16 +21,61 @@ CALLBACK_PREFIX = "telemetry" -_TEXT = ( - "Help improve Condor?\n\n" - "Condor can send an anonymous, allowlisted usage summary to the project: " - "which commands and screens get used, what breaks, and which models agents " - "run. It is off right now and stays off unless you say yes.\n\n" - "Never included: API keys, wallet addresses, server names or URLs, " - "trading pairs, amounts, balances, positions, prompts or agent replies, " - "and no Telegram id or username.\n\n" - "Full details in PRIVACY.md at the root of the repo, which also says how " - "to change or withdraw this answer at any time." +# The two answers, in the order both surfaces offer them. `off` is deliberately +# absent — install counting is the floor (see ``consent.ANSWER_LEVELS``). +OPTIONS = ( + {"level": "usage", "label": "Yes, share usage summaries"}, + {"level": "ping", "label": "Only count my install"}, +) + +# Everything an install is told before it answers. +DISCLOSURE = { + "headline": "Help improve Condor?", + "always_on": ( + "Condor counts installs so the project knows it is used: a random id, " + "the version, and an uptime ping — nothing about you or your trading. " + "That is always on." + ), + "optional": ( + "It can also send an anonymous, allowlisted usage summary: which " + "commands and screens get used, what breaks, and which models agents " + "run. That part is up to you." + ), + # A list in the browser, one sentence in Telegram. The last entry is phrased + # to close the sentence :func:`_never_line` builds. + "never": [ + "API keys", + "wallet addresses", + "server names or URLs", + "trading pairs", + "amounts", + "balances", + "positions", + "prompts or agent replies", + "Telegram id or username", + ], + "doc": ( + "Full details in PRIVACY.md at the root of the repo, which also says " + "how to change this answer at any time." + ), + "options": [dict(option) for option in OPTIONS], +} + + +def _never_line() -> str: + """The ``never`` list as the one sentence Telegram has always sent.""" + items = DISCLOSURE["never"] + return f"Never included: {', '.join(items[:-1])}, and no {items[-1]}." + + +_TEXT = "\n\n".join( + ( + DISCLOSURE["headline"], + DISCLOSURE["always_on"], + DISCLOSURE["optional"], + _never_line(), + DISCLOSURE["doc"], + ) ) @@ -35,15 +86,11 @@ def keyboard(): [ [ InlineKeyboardButton( - "Yes, help improve Condor", callback_data=f"{CALLBACK_PREFIX}:usage" + option["label"], + callback_data=f"{CALLBACK_PREFIX}:{option['level']}", ) - ], - [ - InlineKeyboardButton( - "Only count my install", callback_data=f"{CALLBACK_PREFIX}:ping" - ) - ], - [InlineKeyboardButton("No thanks", callback_data=f"{CALLBACK_PREFIX}:off")], + ] + for option in OPTIONS ] ) @@ -98,23 +145,20 @@ async def callback_handler(update, context) -> None: answer = ( (query.data or "").split(":", 1)[1] if ":" in (query.data or "") else "" ) - if answer == "off": - consent.deny() - await query.edit_message_text( - "Telemetry stays off. Nothing was collected, and the local " - "buffer has been deleted." - ) - return - + # An "off" tap can only come from a prompt sent by an older version; + # grant() lands anything unrecognized on ping, the floor. chosen = consent.grant(answer) from condor.telemetry import emitter - emitter.emit("install") + # Almost always a no-op: boot counted this install already. It stays + # here for the install whose first boot was silenced by + # `CONDOR_TELEMETRY=off` and which is answering after that was lifted. + if consent.mark_install_reported(): + emitter.emit("install") if chosen == consent.PING: await query.edit_message_text( "Thanks. Condor will only report that this install exists and " - "which version it runs. PRIVACY.md says how to change or " - "withdraw this." + "which version it runs. PRIVACY.md says how to change this." ) else: await query.edit_message_text( diff --git a/condor/telemetry/taps.py b/condor/telemetry/taps.py index c32117e55..dac85057c 100644 --- a/condor/telemetry/taps.py +++ b/condor/telemetry/taps.py @@ -94,10 +94,12 @@ def _note_user(hashed: str | None, surface: str) -> None: @_quiet def feature_first_use(feature: str) -> None: """Fire once per install, ever. The activation funnel.""" - if not _on(): - return from condor.telemetry import consent + # Gated on `usage`, not merely "on": at the ping floor the event could + # never be sent, and the features_seen list should not accumulate for it. + if consent.level() != consent.USAGE: + return if consent.mark_feature_seen(feature): emit("feature_first_use", feature=feature) diff --git a/condor/web/app.py b/condor/web/app.py index e5ca6d623..18e7ac323 100644 --- a/condor/web/app.py +++ b/condor/web/app.py @@ -26,6 +26,7 @@ executors, market, meta, + notifications, portfolio, positions, reports, @@ -87,6 +88,7 @@ def create_app() -> FastAPI: app.include_router(market.router, prefix="/api/v1") app.include_router(dex.router, prefix="/api/v1") app.include_router(meta.router, prefix="/api/v1") + app.include_router(notifications.router, prefix="/api/v1") app.include_router(ws.router, prefix="/api/v1") app.include_router(agents.router, prefix="/api/v1") app.include_router(routines.router, prefix="/api/v1") diff --git a/condor/web/auth.py b/condor/web/auth.py index e2986ee01..1ea488fe5 100644 --- a/condor/web/auth.py +++ b/condor/web/auth.py @@ -1,7 +1,5 @@ from __future__ import annotations -import hashlib -import hmac import logging import os import secrets @@ -14,13 +12,11 @@ from condor.web.models import WebUser from config_manager import UserRole, get_config_manager -from utils.config import TELEGRAM_TOKEN logger = logging.getLogger(__name__) _ALGORITHM = "HS256" _TOKEN_EXPIRE_SECONDS = 86400 # 24 hours -_AUTH_WINDOW_SECONDS = 86400 # accept auth_date within 24 hours _LOGIN_TOKEN_TTL = 300 # one-time login tokens valid for 5 minutes _bearer_scheme = HTTPBearer() @@ -46,28 +42,6 @@ def _jwt_secret() -> str: return get_config_manager().get_or_create_web_jwt_secret() -# ── Telegram Login Widget verification ── - - -def verify_telegram_login(data: dict) -> bool: - """Verify data from the Telegram Login Widget using HMAC-SHA256.""" - check_hash = data.get("hash", "") - auth_date = data.get("auth_date", 0) - - # Check auth_date freshness - if abs(time.time() - int(auth_date)) > _AUTH_WINDOW_SECONDS: - return False - - # Build check string (alphabetically sorted key=value, excluding hash) - filtered = {k: v for k, v in data.items() if k != "hash"} - check_string = "\n".join(f"{k}={v}" for k, v in sorted(filtered.items())) - - secret_key = hashlib.sha256(TELEGRAM_TOKEN.encode()).digest() - computed = hmac.new(secret_key, check_string.encode(), hashlib.sha256).hexdigest() - - return hmac.compare_digest(computed, check_hash) - - # ── JWT helpers ── diff --git a/condor/web/models.py b/condor/web/models.py index 757a51a24..5d84cf2ce 100644 --- a/condor/web/models.py +++ b/condor/web/models.py @@ -7,16 +7,6 @@ # ── Auth ── -class LoginRequest(BaseModel): - id: int - first_name: str - last_name: str = "" - username: str = "" - photo_url: str = "" - auth_date: int - hash: str - - class LoginResponse(BaseModel): token: str user: WebUser @@ -38,6 +28,9 @@ class ServerInfo(BaseModel): port: int online: bool = False permission: str = "trader" + # The server this user's commands land on when none is named — the same + # `chat_defaults` entry Telegram reads, so both surfaces agree on it. + is_default: bool = False # ── Portfolio ── @@ -468,6 +461,9 @@ class ReportSummary(BaseModel): source_name: str = "" tags: list[str] = [] agent: str = "" # producing assistant/expert (e.g. "condor", "executor_manager") + # Authenticated owner stamped at save time (SEC-196); None = legacy/ownerless, + # visible to admins only. + user_id: int | None = None class ReportsListResponse(BaseModel): diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index 66753b4e2..5ac8daa26 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -19,15 +19,19 @@ import asyncio import logging -import os import time +from collections import OrderedDict from pathlib import Path from typing import Any from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field -from condor.agents.ownership import OwnedBot +from condor.agents.attribution import ( + apply_bot_mode_pnl, + current_owner_bases, + session_ownership, +) from condor.agents.sessions_index import ( count_experiments, count_sessions, @@ -39,6 +43,7 @@ list_session_snapshots, list_sessions, ) +from condor.fsutil import atomic_write_text from condor.web.auth import check_server_access, get_current_user from condor.web.models import ReportSummary, WebUser @@ -52,7 +57,28 @@ # Only ids that are inactive (no registered engine, not the newest session), # fetched successfully, with open_count == 0, and not in controller mode land # here; everything else keeps flowing through the 30s TTL path above. -_CLOSED_PERF_CACHE: dict[str, Any] = {} +# Bounded LRU (same idiom as condor.web.routes.archived): each entry holds the +# full executor rows, so an unbounded dict would grow with every session ever +# run for the life of the process. Eviction is always safe — a miss just flows +# through the normal fetch path and re-freezes. +_CLOSED_PERF_CACHE_MAX = 256 +_CLOSED_PERF_CACHE: OrderedDict[str, Any] = OrderedDict() + + +def _closed_perf_get(agent_id: str) -> Any | None: + """Return the frozen entry, marking it most-recently-used.""" + perf = _CLOSED_PERF_CACHE.get(agent_id) + if perf is not None: + _CLOSED_PERF_CACHE.move_to_end(agent_id) + return perf + + +def _closed_perf_put(agent_id: str, perf: Any) -> None: + """Store a frozen entry, evicting the least-recently-used past the cap.""" + _CLOSED_PERF_CACHE[agent_id] = perf + _CLOSED_PERF_CACHE.move_to_end(agent_id) + while len(_CLOSED_PERF_CACHE) > _CLOSED_PERF_CACHE_MAX: + _CLOSED_PERF_CACHE.popitem(last=False) def _cache_get(key: str) -> Any | None: @@ -409,230 +435,12 @@ async def _get_client_for_strategy(strategy_dir: Path, default_config: dict | No return client, server_name -def _session_bot_base(strategy_dir: Path, default_config: dict | None, num: int) -> str: - """Bot base name a session operates: per-session config, else strategy default. - - A non-empty per-session ``bot_name`` wins (so runtime-named bots that record - their deployed name resolve), but an empty/absent one falls back to the - strategy default — early sessions predating the config's ``bot_name`` saved it - as ``''`` and must still map to the shared bot they operated. Empty string when - neither is set (direct-executor strategies). Shared by the per-session PnL - distribution and the operator's live-executor view so both resolve identically. - """ - from condor.agents.config import load_full_config - from condor.agents.sessions_index import find_session_dir - - default_base = (default_config or {}).get("bot_name", "") or "" - sd = find_session_dir(strategy_dir, num) - if not sd: - return default_base - return load_full_config(sd, default_config).get("bot_name", "") or default_base - - -def _session_start_epoch(strategy_dir: Path, num: int) -> float: - """Session start time: config.yml is written once at start, so its mtime is stable.""" - from condor.agents.sessions_index import find_session_dir - - sd = find_session_dir(strategy_dir, num) - if not sd: - return 0.0 - cfg = sd / "config.yml" - target = cfg if cfg.exists() else sd - try: - return os.path.getmtime(target) - except OSError: - return 0.0 - - -def _session_ownership( - strategy_dir: Path, default_config: dict | None, num: int -) -> list[OwnedBot]: - """Bases a session owned and the instant it took each over, oldest first. - - Two sources, in order: - - 1. ``{session_dir}/owned_bots.json`` — the ledger [[FEAT-017]] writes, which - knows both the bases (a session may operate several) and the exact takeover - instant, whether the bot was deployed here or adopted after a restart. - 2. the legacy shim — a session predating the ledger resolves its single - ``bot_name`` as one owned bot ``since`` the session started, reproducing the - session-start tiling attribution used before the ledger existed. - - Empty for direct-executor strategies, whose per-session executor attribution - already stands and must not be touched. - """ - from condor.agents.ownership import read_owned - - owned = read_owned(find_session_dir(strategy_dir, num)) - if owned: - return owned - base = _session_bot_base(strategy_dir, default_config, num) - if not base: - return [] - start = _session_start_epoch(strategy_dir, num) - return [OwnedBot(base=base, origin="legacy", since=start, last_seen=start)] - - -def _owner_windows( - real_sessions: list, strategy_dir: Path, default_config: dict | None -) -> dict[str, list[tuple[float, Any, float]]]: - """``{base: [(since, session, until), …]}`` — owners, oldest takeover first. - - The windows a base's owners occupy tile ``[since_i, since_{i+1})`` and the last - one runs to now, so slicing over them reproduces the bot's whole cumulative with - no gap and no double count. Keyed per base rather than globally per session - number: two bases handed over at different moments never share a timeline. - - ``until`` is the instant the session released the bot, or ``0.0`` while it - still holds it. A released last window stops there rather than running to now, - which is the one case where the tiling deliberately leaves a gap: PnL a bot - earned with no session operating it belongs to no session. - """ - owners: dict[str, list[tuple[float, Any, float]]] = {} - for s in real_sessions: - for ob in _session_ownership(strategy_dir, default_config, s.session_num): - owners.setdefault(ob.base, []).append((ob.since, s, ob.until)) - for lst in owners.values(): - lst.sort(key=lambda t: (t[0], t[1].session_num)) - return owners - - -def _current_owner_bases( - strategy_dir: Path, - default_config: dict | None, - session_nums: list[int], - num: int, -) -> list[str]: - """Bases ``num`` is the CURRENT owner of — the last takeover by ``since``. - - A bot's live open positions belong to whoever operates it now, so this is the - gate for merging them into one session's view. Same rule - :func:`_apply_bot_mode_pnl` applies to live unrealized PnL, kept here as one - lookup over the same windows so the rollup and the per-session detail can - never disagree about who holds the open book. A session that released the bot - is not its current owner, so an ended session shows no live open book. - """ - last: dict[str, tuple[float, int, float]] = {} - for n in session_nums: - for ob in _session_ownership(strategy_dir, default_config, n): - if last.get(ob.base, (float("-inf"), -1, 0.0))[:2] <= (ob.since, n): - last[ob.base] = (ob.since, n, ob.until) - return sorted( - base for base, (_, owner, until) in last.items() if owner == num and until <= 0 - ) - - -async def _apply_bot_mode_pnl( - real_sessions: list, strategy_dir: Path, default_config: dict | None, client: Any -) -> None: - """Distribute each owned bot's PnL across the sessions that operated it. - - One rule covers deploy and handover: every owned bot is attributed by slicing - its history over ``[since, next_owner.since or now)``, where ``since`` is the - takeover instant the ownership ledger recorded. A bot the session *deployed* - has no history before its ``since``, so the general rule already hands it the - whole instance — the exact case falls out instead of needing its own branch. - - Live unrealized PnL, fees and open positions go to each base's LAST owner by - ``since`` — a lookup in the ledger where it used to be a ``max(session_num)`` - guess, so a new session that never adopted the bot no longer inherits its - open book. - - Works uniformly for single- and multi-controller bots (history sums controllers - per instance) and for a base re-launched under several instances. Strategies - whose sessions own no bot (direct-executor agents) are left untouched. - """ - from condor.fetchers.bot_performance import ( - bot_executor_rows, - fetch_all_bot_performance, - fetch_archived_instances, - fetch_base_histories, - resolve_bots, - slice_history, - ) - - if not client or not real_sessions: - return - owners = _owner_windows(real_sessions, strategy_dir, default_config) - bases = sorted(owners) - if not bases: - return # direct-executor strategy — nothing to attribute - - try: - all_perf = await fetch_all_bot_performance(client) - except Exception as e: - log.warning("bot perf fetch for %s failed: %s", strategy_dir.name, e) - return - - now = time.time() - # The oldest takeover across every base sets how far back the histories must - # reach; sampling resolution is chosen from it so no owner's window falls off - # the end of the retained rows. - earliest = min( - (since for lst in owners.values() for since, _, _ in lst if since > 0), - default=0.0, - ) - # Archived instances carry the realized PnL of every bot a session stopped — - # the normal end state of a finished session, and invisible in the live - # snapshot. Same universe the live agent's own view uses, so the dashboard and - # the tick loop cannot disagree about what a session earned. - archived = await fetch_archived_instances(client) - histories_by_base = await fetch_base_histories( - client, all_perf, bases, earliest, now, extra_names=archived - ) - - live = resolve_bots(all_perf, bases) - for base in bases: - window_owners = owners[base] - insts = histories_by_base.get(base, []) - - # Realized / volume / trades / fees: one window per owner, tiling the - # timeline. A released window (the session stopped and left the bot - # running) ends at its release instant, so PnL earned while no session - # was operating the bot is attributed to nobody instead of accruing to - # whoever happened to hold last. - sliced_fees = 0.0 - for i, (since, s, until) in enumerate(window_owners): - end = window_owners[i + 1][0] if i + 1 < len(window_owners) else now - if until > 0: - end = min(end, until) - if end <= since: - continue - realized, volume, trades, fees = slice_history(insts, since, end) - s.realized_pnl += realized - s.volume += volume - # Sliced closes are round-trip closes, so they are this session's - # trades AND its closed positions — the same two counters - # _merge_bot_perf bumps, so the session detail and this rollup report - # one number. - s.trade_count += int(round(trades)) - s.closed_count += int(round(trades)) - s.fees += fees - sliced_fees += fees - # Volume with no fee is a missing column, not a free trade. - if volume > 0 and fees == 0.0: - s.fees_known = False - s.total_pnl = s.realized_pnl + s.unrealized_pnl - - # Live unrealized + open positions → the base's current owner, unless it - # has released the bot: an ended session holds no open book. - bot = live.get(base) - if not bot: - continue - last_since, operator, last_until = window_owners[-1] - if last_until > 0: - continue - b_rows = bot_executor_rows(bot) - operator.unrealized_pnl += float(bot.get("unrealized_pnl_quote", 0) or 0) - # Fees come from the sliced history when the backend reports a cumulative - # figure. When it does not, that column is all zeros and the only fees - # available are the live open-position ones — attributed to the current - # operator as before rather than silently dropped. - if sliced_fees == 0.0: - operator.fees += float(bot.get("cum_fees_quote", 0) or 0) - operator.open_count += sum(1 for r in b_rows if r["status"] == "RUNNING") - operator.executors = list(operator.executors) + b_rows - operator.total_pnl = operator.realized_pnl + operator.unrealized_pnl +# ── Session-ownership PnL attribution ── +# The whole engine — session ownership resolution, owner-window tiling, the +# slice-and-merge fee rules, and the current-owner rule for the live open book — +# lives in condor.agents.attribution ([[ARCH-191]]), shared with the agent's own +# view (condor.agents.performance) so the dashboard and the tick loop compute +# from one implementation. This module keeps only HTTP concerns and the caches. async def _compute_strategy_performance( @@ -655,7 +463,7 @@ async def _compute_strategy_performance( # Per-session executor fetches stay bot-free (bot_names=None below) so closed # sessions can be frozen; bot-mode PnL is distributed per session afterward by - # _apply_bot_mode_pnl from the controller history, which handles both fixed and + # apply_bot_mode_pnl from the controller history, which handles both fixed and # runtime-named (per-session config) bots. sessions: list[AgentPerformanceModel] = [] if client and ids: @@ -677,11 +485,17 @@ async def _compute_strategy_performance( for aid in active_ids: _CLOSED_PERF_CACHE.pop(aid, None) - fetch_ids = [ - aid - for aid, _, _ in ids - if aid in active_ids or aid not in _CLOSED_PERF_CACHE - ] + # Snapshot the frozen entries up front (marking them recently-used) so + # an LRU eviction during the await below can't drop a session from + # this render. + frozen: dict[str, Any] = {} + for aid, _, _ in ids: + if aid not in active_ids: + perf = _closed_perf_get(aid) + if perf is not None: + frozen[aid] = perf + + fetch_ids = [aid for aid, _, _ in ids if aid in active_ids or aid not in frozen] perf_map: dict[str, Any] = {} failed_ids: set[str] = set() @@ -698,7 +512,7 @@ async def _compute_strategy_performance( for agent_id, num, kind in ids: perf = perf_map.get(agent_id) if perf is None: - perf = _CLOSED_PERF_CACHE.get(agent_id) + perf = frozen.get(agent_id) if perf is None: continue # Freeze immutable results: fetched fine, no engine, not the newest @@ -711,7 +525,7 @@ async def _compute_strategy_performance( and agent_id not in failed_ids and perf.open_count == 0 ): - _CLOSED_PERF_CACHE[agent_id] = perf + _closed_perf_put(agent_id, perf) if kind == "experiment" and perf.trade_count == 0: continue sessions.append( @@ -740,7 +554,7 @@ async def _compute_strategy_performance( # bot is DISTRIBUTED — not duplicated — the totals below are a plain additive # sum of the rows and stay correct for both modes with no double counting. if client and real_sessions: - await _apply_bot_mode_pnl(real_sessions, strategy_dir, default_config, client) + await apply_bot_mode_pnl(real_sessions, strategy_dir, default_config, client) totals = { "total_pnl": sum(s.total_pnl for s in real_sessions), @@ -1146,7 +960,7 @@ async def update_agent_md( ): """Update AGENT.md content.""" agent = _get_agent(slug) - (agent.agent_dir / "AGENT.md").write_text(req.content) + atomic_write_text(agent.agent_dir / "AGENT.md", req.content) return {"updated": True} @@ -1160,8 +974,8 @@ async def update_agent_config( write the MCP ``manage_trading_agent`` tool already performs — the web layer simply had no door to it, which is why the UI could only offer a text editor. """ + from condor.llm.options import AGENT_OPTIONS from config_manager import get_config_manager - from handlers.agents._shared import AGENT_OPTIONS agent = _get_agent(slug) @@ -1210,6 +1024,57 @@ async def delete_agent(slug: str, user: WebUser = Depends(get_current_user)): return {"deleted": True} +def _chat_member_status(member: Any) -> str | None: + """Normalize the two shapes ``get_chat_member`` can answer with. + + A live python-telegram-bot returns a ``ChatMember`` with a ``status`` + attribute; ``_HttpBot`` hands back Telegram's raw envelope (or ``None`` + when it has no token). ``None`` means "could not verify". + """ + if member is None: + return None + if isinstance(member, dict): + if not member.get("ok"): + return None + return (member.get("result") or {}).get("status") + return getattr(member, "status", None) + + +async def _check_chat_access(user_id: int, chat_id: int) -> None: + """403 unless ``chat_id`` is a chat the caller actually belongs to (SEC-198). + + The routes below forward a body-supplied ``chat_id`` to outbound Telegram + sends, so an unchecked value lets any authenticated session speak with the + bot's identity into anyone's chat. A private chat's id *is* the Telegram + user id, so the common case (and the MCP crossback for private sessions) + costs nothing; any other id must be a group the caller is a member of, + verified against Telegram itself through the same bot ladder that would + deliver the message. Verification failure fails closed: an unverifiable + target is a refused target. Admins are exempt, mirroring the delegation + ownership gate (SEC-081). + """ + if not chat_id or chat_id == user_id: + return + from config_manager import get_config_manager + + if get_config_manager().is_admin(user_id): + return + from condor.agents.delegate import resolve_bot + + status = None + try: + member = await resolve_bot().get_chat_member(chat_id=chat_id, user_id=user_id) + status = _chat_member_status(member) + except Exception: + log.warning( + "Could not verify membership of user %s in chat %s", user_id, chat_id + ) + if status is None or status in ("left", "kicked", "banned"): + raise HTTPException( + status_code=403, detail="chat_id is not a chat you belong to" + ) + + @router.post("/{slug}/consult") async def consult_agent( slug: str, req: ConsultRequest, user: WebUser = Depends(get_current_user) @@ -1229,6 +1094,10 @@ async def consult_agent( if req.server_name: check_server_access(user.id, req.server_name) + # The chat is where the consult's notifications land — same ownership rule + # as the push target on /notify (SEC-198). + await _check_chat_access(user.id, req.chat_id) + # Web callers always act as themselves; the ``user_id`` override is reserved # for trusted internal/MCP callers and must not let a session impersonate # another user's memory/skill scope. @@ -1287,6 +1156,11 @@ async def delegate_agent( if req.server_name: check_server_access(user.id, req.server_name) + # ``chat_id`` is where ``_notify_done`` will push the completion text, so a + # foreign chat here would let the delegation's summary (driven by the + # caller's task text) land in someone else's chat (SEC-198). + await _check_chat_access(user.id, req.chat_id) + conversation_id = await _conversation_for_session(req.session_key) # Depth 1, structurally. A delegate worker cannot delegate at all @@ -1379,6 +1253,11 @@ async def notify_user(req: NotifyRequest, user: WebUser = Depends(get_current_us if not req.text: raise HTTPException(status_code=400, detail="text is required") + # The push target must belong to the caller — mirror the ``req.user_id`` + # rule below for the outbound address, and refuse before any side effect + # (SEC-198). + await _check_chat_access(user.id, req.chat_id) + # The caller is the JWT, never ``req.user_id``: mirror consult/delegate so an # authenticated session cannot write into another user's transcript. conversation_id = await _conversation_for_session(req.session_key) @@ -1396,6 +1275,19 @@ async def notify_user(req: NotifyRequest, user: WebUser = Depends(get_current_us exc_info=True, ) + # The dashboard bell (FEAT-048), addressed to the caller themselves — never + # to ``req.chat_id``, which may legitimately be a group they belong to but + # which has no dashboard owner. This is what makes ``send_notification`` + # succeed on an install with no Telegram: the tool already counts + # ``recorded`` as delivered. + try: + from condor.notifications import record + + if await record(user.id, req.text, kind="agent"): + recorded = True + except Exception: + log.debug("Could not record a notification for user %s", user.id, exc_info=True) + sent = False if req.chat_id: sent = await _push_to_telegram(req.chat_id, req.text, req.parse_mode) @@ -1441,8 +1333,9 @@ async def create_strategy( learnings_path = strategy.dir / "learnings.md" if not learnings_path.exists(): - learnings_path.write_text( - "# Learnings\n\n## Active Insights\n\n## Retired Insights\n" + atomic_write_text( + learnings_path, + "# Learnings\n\n## Active Insights\n\n## Retired Insights\n", ) return StrategySummary( @@ -1541,7 +1434,7 @@ async def update_strategy_md( ): """Update strategy.md content.""" strategy = _get_strategy(slug, sslug) - (strategy.dir / "strategy.md").write_text(req.content) + atomic_write_text(strategy.dir / "strategy.md", req.content) return {"updated": True} @@ -1629,7 +1522,7 @@ async def get_session_executors( # Bot-mode: the session operates named bots whose executors live in the bot # container, not the agent_id-keyed table. Merge the live positions of every # base this session CURRENTLY owns — the same last-owner-by-`since` rule - # _apply_bot_mode_pnl uses, so the two views never disagree. A session that + # apply_bot_mode_pnl uses, so the two views never disagree. A session that # handed its bot over shows only its own direct executors; the live open book # belongs to whoever operates the bot now. session_nums = [ @@ -1637,13 +1530,13 @@ async def get_session_executors( for _, n, k in enumerate_agent_ids(_runkey(slug, sslug), strategy.dir) if k == "session" ] - bot_names = _current_owner_bases( + bot_names = current_owner_bases( strategy.dir, strategy.default_config, session_nums, session_num ) # Slice the bot to this session's window for the same reason the rollup does: # merging the lifetime aggregate here made the session detail disagree with # the session's own row in the strategy list. - owned = _session_ownership(strategy.dir, strategy.default_config, session_num) + owned = session_ownership(strategy.dir, strategy.default_config, session_num) since = min((b.since for b in owned if b.since > 0), default=0.0) perf = await fetch_agent_performance( client, agent_id, bot_names=bot_names, since=since @@ -1739,6 +1632,10 @@ async def _start(agent, strategy, req: StartStrategyRequest, user_id: int) -> di if req.config: config_dict.update(req.config) + # The engine notifies ``chat_id`` on every tick, so the same ownership rule + # as /notify applies to it (SEC-198). + await _check_chat_access(user_id, req.chat_id) + # ``TickEngine._resolve_server`` trades on ``config["server_name"]`` and the # request body is a free-form dict, so without this gate any authenticated # user could start a live loop on another user's stored credentials — the @@ -1912,7 +1809,7 @@ async def update_learnings( ): """Update a strategy's learnings.md.""" strategy = _get_strategy(slug, sslug) - (strategy.dir / "learnings.md").write_text(req.content) + atomic_write_text(strategy.dir / "learnings.md", req.content) return {"updated": True} diff --git a/condor/web/routes/archived.py b/condor/web/routes/archived.py index d65539fa0..de62ea9f6 100644 --- a/condor/web/routes/archived.py +++ b/condor/web/routes/archived.py @@ -3,6 +3,7 @@ import asyncio import logging import os +from collections import OrderedDict from typing import Any from fastapi import APIRouter, Depends, HTTPException, Query @@ -23,10 +24,36 @@ router = APIRouter(tags=["archived"]) -# Module-level cache for archived data (immutable, no TTL needed) -# Key: (server_name, db_path) -> cached result -_performance_cache: dict[tuple[str, str], ArchivedBotPerformance] = {} -_executors_cache: dict[tuple[str, str], list[NormalizedExecutor]] = {} +# Module-level LRU cache for archived data (immutable per db_path, so no TTL — +# but bounded, since each entry holds up to ~5000 PnL points plus the full +# executor list and a long-lived process would otherwise grow without ceiling). +# Key: (server_name, db_path) -> cached result. Executors live only inside the +# performance entry; the executors route pages out of it. +_PERFORMANCE_CACHE_MAX = 32 +_performance_cache: OrderedDict[tuple[str, str], ArchivedBotPerformance] = OrderedDict() + +# In-flight fetches keyed like the cache, so concurrent cold-cache requests for +# the same archived bot share one backend walk (same idiom as +# condor.pool_data._single_flight). +_performance_inflight: dict[tuple[str, str], "asyncio.Task[ArchivedBotPerformance]"] = ( + {} +) + + +def _cache_get(cache_key: tuple[str, str]) -> ArchivedBotPerformance | None: + """Return the cached entry, marking it most-recently-used.""" + perf = _performance_cache.get(cache_key) + if perf is not None: + _performance_cache.move_to_end(cache_key) + return perf + + +def _cache_put(cache_key: tuple[str, str], perf: ArchivedBotPerformance) -> None: + """Store an entry, evicting the least-recently-used past the cap.""" + _performance_cache[cache_key] = perf + _performance_cache.move_to_end(cache_key) + while len(_performance_cache) > _PERFORMANCE_CACHE_MAX: + _performance_cache.popitem(last=False) def _extract_bot_name(db_path: str) -> str: @@ -182,12 +209,36 @@ def _derive_primary_pair( async def _fetch_and_cache_performance( client: Any, name: str, db_path: str ) -> ArchivedBotPerformance: - """Fetch full performance data and cache it.""" + """Return cached performance data, coalescing concurrent cold-cache fetches. + + The fetch runs as a detached task and awaiters ``shield`` it, so one caller + navigating away and cancelling their request cannot also cancel the fetch + the remaining callers are waiting on. + """ cache_key = (name, db_path) - # Check cache first - if cache_key in _performance_cache: - return _performance_cache[cache_key] + cached = _cache_get(cache_key) + if cached is not None: + return cached + + task = _performance_inflight.get(cache_key) + if task is None or task.done(): + task = asyncio.ensure_future(_fetch_performance(client, name, db_path)) + _performance_inflight[cache_key] = task + + def _clear(finished: "asyncio.Task", _key: tuple[str, str] = cache_key) -> None: + if _performance_inflight.get(_key) is finished: + _performance_inflight.pop(_key, None) + + task.add_done_callback(_clear) + return await asyncio.shield(task) + + +async def _fetch_performance( + client: Any, name: str, db_path: str +) -> ArchivedBotPerformance: + """Fetch full performance data from the backend and cache it.""" + cache_key = (name, db_path) # Fetch summary try: @@ -237,9 +288,6 @@ async def _fetch_and_cache_performance( # Normalize executors to consistent field names executors = _normalize_executors(raw_executors) - # Cache executors separately for pagination - _executors_cache[cache_key] = executors - # Derive primary connector and trading pair primary_connector, primary_trading_pair = _derive_primary_pair( executors, exchanges, trading_pairs @@ -303,8 +351,8 @@ async def _fetch_and_cache_performance( executor_count=len(executors), ) - # Cache full result - _performance_cache[cache_key] = result + # Cache full result (executors included — the executors route pages out of it) + _cache_put(cache_key, result) return result @@ -382,12 +430,14 @@ async def get_archived_executors( cache_key = (name, db_path) - # If executors not cached yet, trigger a full fetch - if cache_key not in _executors_cache: + # Page out of the cached performance entry; on a miss, trigger the full + # (single-flight) fetch. + perf = _cache_get(cache_key) + if perf is None: client = await cm.get_client(name) - await _fetch_and_cache_performance(client, name, db_path) + perf = await _fetch_and_cache_performance(client, name, db_path) - executors = _executors_cache.get(cache_key, []) + executors = perf.executors page = executors[offset : offset + limit] return PaginatedExecutors( diff --git a/condor/web/routes/auth.py b/condor/web/routes/auth.py index 5eb6703d2..3c2a61b91 100644 --- a/condor/web/routes/auth.py +++ b/condor/web/routes/auth.py @@ -6,6 +6,7 @@ from condor.web.auth import create_jwt, get_current_user, redeem_login_token from condor.web.models import LoginResponse, WebUser from config_manager import UserRole, get_config_manager +from utils import config as app_config router = APIRouter(tags=["auth"]) @@ -55,6 +56,67 @@ async def token_login(req: TokenLoginRequest, response: Response): return LoginResponse(token=jwt_token, user=user) +@router.get("/auth/mode") +async def auth_mode() -> dict: + """How this install logs in. Unauthenticated on purpose. + + The Login page has to know *how* to log in before it can, and the answer is + a property of the install, not of any user: either "run /web in Telegram" or + "you are already home". It leaks nothing a visitor could not learn by trying + :func:`local_login` and reading the status code. + """ + return {"mode": app_config.CONDOR_MODE} + + +@router.post("/auth/local-login", response_model=LoginResponse) +async def local_login(response: Response): + """Mint a session for the local admin. Local mode only (FEAT-049). + + Deliberately the *only* mode-aware place in the auth path. Once this returns, + the JWT, the WS handshake, ``check_server_access``, roles and all 200-odd + guarded endpoints behave identically to Telegram mode, because they are + identical — ``get_current_user`` never learns which mode it is in. Outside + local mode this endpoint does not exist: 404, not 403, because "there is a + passwordless login here, you just may not use it" is not a thing worth + telling anyone. + + The control that keeps this safe is not this check alone — local mode also + binds loopback only (``utils.config.resolve_web_host``). + """ + if not app_config.LOCAL_MODE: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") + + # Same headers as token_login: never leak the issued JWT via Referer, and + # never let a proxy or the browser cache a session response. + response.headers["Referrer-Policy"] = "no-referrer" + response.headers["Cache-Control"] = "no-store" + + # Who this is was settled at boot by ``utils.config.check_local_user``, which + # refuses to start an install this would 500 on. Reaching it here means the + # user was removed or blocked while the process was up, so the message names + # the knob that decides it rather than sending anyone back to the wizard. + user_id = app_config.LOCAL_USER_ID + role = get_config_manager().get_user_role(user_id) + if role not in (UserRole.USER, UserRole.ADMIN): + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=( + f"Local mode logs in as user {user_id} (ADMIN_USER_ID in .env), " + "who is no longer an approved user in config.yml. Restore that " + "user, or run `make setup` and choose Local mode." + ), + ) + + user = WebUser(id=user_id, username="local", first_name="Local", role=role.value) + token = create_jwt( + user_id=user_id, + username=user.username, + first_name=user.first_name, + role=role.value, + ) + return LoginResponse(token=token, user=user) + + @router.get("/auth/me", response_model=WebUser) async def me(user: WebUser = Depends(get_current_user)): return user diff --git a/condor/web/routes/backtesting.py b/condor/web/routes/backtesting.py index ff35163f6..89b944144 100644 --- a/condor/web/routes/backtesting.py +++ b/condor/web/routes/backtesting.py @@ -121,9 +121,9 @@ async def get_backtest_task( except Exception: pass - # Fallback to saved + # Fallback to saved (only if the result belongs to this server) saved = store.get_result(task_id) - if saved: + if saved and saved.get("server") == name: return normalize_backtest_task({**saved, "saved": True}) raise HTTPException(status_code=404, detail="Task not found") @@ -138,6 +138,10 @@ async def delete_backtest_task( cm = get_config_manager() store = get_backtest_store() + saved = store.get_result(task_id) + if saved is not None and saved.get("server") != name: + # Result belongs to another server; don't let this server delete it + raise HTTPException(status_code=404, detail="Task not found") store.delete_result(task_id) # Also try to delete from live @@ -169,7 +173,8 @@ async def delete_saved_result( ): store = get_backtest_store() - deleted = store.delete_result(task_id) - if not deleted: + saved = store.get_result(task_id) + if saved is None or saved.get("server") != name: raise HTTPException(status_code=404, detail="Saved result not found") + store.delete_result(task_id) return {"deleted": True} diff --git a/condor/web/routes/bots.py b/condor/web/routes/bots.py index 996f64b52..fe06c2b7e 100644 --- a/condor/web/routes/bots.py +++ b/condor/web/routes/bots.py @@ -8,6 +8,7 @@ import yaml from fastapi import APIRouter, Depends, HTTPException +from condor.controller_configs import clean_config_for_save from condor.fetchers.bots import build_bots_page, extract_bots_list from condor.web.auth import require_server_access from condor.web.models import ( @@ -24,7 +25,6 @@ ) from condor.web.routes._errors import upstream_error from config_manager import get_config_manager -from handlers.bots._shared import clean_config_for_save logger = logging.getLogger(__name__) diff --git a/condor/web/routes/chat_ws.py b/condor/web/routes/chat_ws.py index ecb632e3d..2f9719735 100644 --- a/condor/web/routes/chat_ws.py +++ b/condor/web/routes/chat_ws.py @@ -13,6 +13,9 @@ from fastapi import APIRouter, Depends, Query, WebSocket, WebSocketDisconnect +from condor.llm.openrouter_models import fetch_models +from condor.llm.options import DEFAULT_AGENT +from condor.notifications import Notification, register_push_sink from condor.runtime import WEB, EventType, PromptRequest, SessionKey, SessionSpec from condor.runtime import client as runtime from condor.runtime import conversations @@ -27,8 +30,6 @@ from condor.runtime.wake import register_note_sink, register_sink_factory from condor.web.auth import decode_jwt, extract_ws_token, get_current_user from condor.web.models import WebUser -from handlers.agents._shared import DEFAULT_AGENT -from handlers.agents.openrouter_models import fetch_models log = logging.getLogger(__name__) @@ -295,10 +296,37 @@ async def _deliver_note( await _send(ws, message) +async def _push_notification(notification: Notification) -> None: + """Light this user's bell in every tab they have open (FEAT-048). + + A third out-of-band push on this socket, next to ``permission_request`` and + ``system_note``. Unlike those two it is addressed to the *user*, not to a + conversation — a finished background task has no slot — so the frame + carries no ``slot_id`` and the store, not this push, is what guarantees the + notice is seen: a user with no tab open simply has no entry here, and the + bell picks it up from ``GET /notifications`` on the next load. + """ + for ws in list(_attached_sockets.get(notification.user_id, ())): + await _send( + ws, + { + "event": "notification", + "id": notification.id, + "kind": notification.kind, + "title": notification.title, + "text": notification.text, + "link": notification.link, + "ts": notification.ts, + }, + ) + + # Registered here rather than imported by the runtime: ``condor.runtime`` must -# not depend on web or handler code (see ``client._local()``). +# not depend on web or handler code (see ``client._local()``). ``condor.notifications`` +# is registered the same way and for the same reason. register_sink_factory(WEB, _wake_sink) register_note_sink(WEB, _deliver_note) +register_push_sink(_push_notification) @router.websocket("/ws/chat") diff --git a/condor/web/routes/dex.py b/condor/web/routes/dex.py index 3c5781dc8..bba17b46f 100644 --- a/condor/web/routes/dex.py +++ b/condor/web/routes/dex.py @@ -371,10 +371,100 @@ async def ensure_dex_tokens( return {"tokens": verdicts} +class AddTokenRequest(BaseModel): + """One token the user explicitly asked to put on Gateway's list.""" + + network: str = Field(description="Gateway network id, e.g. solana-mainnet-beta") + address: str = Field(description="Token mint/contract to register") + symbol: str | None = Field( + default=None, + description="Ticker the pool shows for it, used to name a conflicting holder", + ) + replace: bool = Field( + default=False, + description="Delete the token currently holding the ticker before adding", + ) + + +@router.post("/servers/{name}/dex/tokens/add") +async def add_dex_token( + name: str, + body: AddTokenRequest, + user: WebUser = Depends(require_server_access), +): + """Register one token because the user clicked a button asking for it. + + The automatic path (``/dex/tokens``, above) reports rather than forces: a + ``failed`` verdict is a Gateway hiccup, and ``symbol_taken`` means another + token already owns the ticker — a conflict only a human should resolve, + because deleting the holder changes what that ticker means everywhere. + This route is that human's answer. Without ``replace`` it retries the + registration and, on a collision, names the holder so the browser can ask; + with ``replace`` it deletes the holder first and registers the new token in + its place. + + Unlike the ensure route, an upstream error here is raised, not folded into + a verdict: the user pressed a button and deserves the actual reason. + """ + from condor.fetchers.gateway_tokens import ( + ensure_tokens_listed, + find_symbol_holder, + forget_listed, + token_addresses, + ) + from condor.pool_data import GECKO_TO_GATEWAY_NETWORK, NETWORK_TO_GECKO + + gateway_network = GECKO_TO_GATEWAY_NETWORK.get( + NETWORK_TO_GECKO.get(body.network, "") + ) + if not gateway_network: + raise HTTPException(status_code=400, detail="Unknown network") + + addresses = token_addresses(body.address) + if not addresses: + raise HTTPException(status_code=400, detail="Not a token address") + address = addresses[0] + + cm = get_config_manager() + client = await cm.get_client(name) + try: + if body.replace and body.symbol: + holder = await find_symbol_holder( + client, gateway_network, body.symbol, exclude=address + ) + if holder: + await client.gateway.delete_token(gateway_network, holder["address"]) + forget_listed(gateway_network, holder["address"]) + + verdicts = await ensure_tokens_listed(client, gateway_network, [address]) + verdict = verdicts.get(address, "failed") + + # The collision's other half: who holds the ticker. Looked up only when + # the answer is needed, and only nameable when the pool knows its own + # ticker — a `???` pool gets the verdict without the culprit. + conflict = None + if verdict == "symbol_taken" and body.symbol: + conflict = await find_symbol_holder( + client, gateway_network, body.symbol, exclude=address + ) + except Exception as e: + logger.warning("add token %s on %s failed: %s", address, gateway_network, e) + raise HTTPException(status_code=502, detail=f"Gateway refused: {e}") from e + + # `ensure_tokens_listed` folds its own upstream errors into this verdict so + # the automatic path stays quiet; a clicked button must not. + if verdict == "failed": + raise HTTPException( + status_code=502, + detail="Gateway could not register the token — is it running?", + ) + return {"verdict": verdict, "conflict": conflict} + + @router.get("/servers/{name}/dex/pools-by-address") async def list_pools_by_address( name: str, - addresses: str = Query(description="Comma-separated pool addresses (max 30)"), + addresses: str = Query(description="Comma-separated pool addresses"), network: str = Query(default="solana-mainnet-beta"), user: WebUser = Depends(require_server_access), ): @@ -385,13 +475,17 @@ async def list_pools_by_address( whenever it was starred. An address that no longer resolves is absent from the answer instead of failing the whole view — a de-indexed pool is not an error. + No cap here: ``fetch_pools_by_addresses`` already splits the list per + GeckoTerminal's 30-address maximum and caches per batch, so truncating would + only drop favourites silently — the 31st star would never render anywhere. + Declared above ``/dex/pools/{pool_address}`` deliberately: a path segment would otherwise be read as an address. """ from condor.pool_data import fetch_pools_by_addresses - wanted = [a.strip() for a in (addresses or "").split(",") if a.strip()][:30] + wanted = [a.strip() for a in (addresses or "").split(",") if a.strip()] if not wanted: return {"pools": [], "upstream": _upstream_state()} before = _throttle_counter() diff --git a/condor/web/routes/notifications.py b/condor/web/routes/notifications.py new file mode 100644 index 000000000..61e5d7ee6 --- /dev/null +++ b/condor/web/routes/notifications.py @@ -0,0 +1,48 @@ +"""Task notifications as an addressable resource (FEAT-048). + +The WS ``notification`` event (``chat_ws.py``) is the low-latency path; this is +the durable one, so a notice raised while no tab was open is still in the bell +on the next page load. + +Everything here is scoped to ``user.id`` from the JWT and never to a +body-supplied id — a notification is addressed to a person, and the store is the +only thing standing between one person's finished tasks and another's. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel + +from condor import notifications +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +router = APIRouter(prefix="/notifications", tags=["notifications"]) + + +class MarkReadRequest(BaseModel): + """Which notifications to mark read. ``None`` (the default) means all.""" + + ids: list[str] | None = None + + +@router.get("") +async def list_notifications( + limit: int = Query(default=50, ge=1, le=100), + user: WebUser = Depends(get_current_user), +): + """This user's notifications, newest first, with the unread count.""" + items = notifications.list_for(user.id, limit=limit) + return { + "items": [n.to_wire() for n in items], + "unread": notifications.unread_count(user.id), + } + + +@router.post("/read") +async def mark_notifications_read( + body: MarkReadRequest, user: WebUser = Depends(get_current_user) +): + """Mark some (or all) of this user's notifications read.""" + return {"unread": notifications.mark_read(user.id, body.ids)} diff --git a/condor/web/routes/portfolio.py b/condor/web/routes/portfolio.py index 19b5829e6..82bae2f63 100644 --- a/condor/web/routes/portfolio.py +++ b/condor/web/routes/portfolio.py @@ -391,8 +391,3 @@ def _extract_connector_totals(data: object) -> dict[str, float]: elif isinstance(val, (int, float)): totals[account] = float(val) return totals - - -def _sum_snapshot_value(data: object) -> float: - """Sum USD values from a portfolio snapshot (nested account/connector/balances).""" - return sum(_extract_connector_totals(data).values()) diff --git a/condor/web/routes/positions.py b/condor/web/routes/positions.py index d7baee46a..e6d00c09c 100644 --- a/condor/web/routes/positions.py +++ b/condor/web/routes/positions.py @@ -22,6 +22,14 @@ def _normalize_position(pos: dict, source: str, source_name: str) -> dict: notional_value = abs(float(amount)) * float(entry_price) except (ValueError, TypeError): notional_value = 0 + # Which executors this row aggregates. The only attribution the API gives a + # held position: a position has no pool of its own, but an LP executor + # records the pool it opened in, so the pool workspace can tell a hold that + # belongs to the pool on screen from one held in another pool of the same + # pair on the same chain. + executor_ids = pos.get("executor_ids") or [] + if not isinstance(executor_ids, list): + executor_ids = [] return { "connector_name": pos.get("connector_name") or pos.get("connector") or "", "trading_pair": pos.get("trading_pair") or "", @@ -38,6 +46,7 @@ def _normalize_position(pos: dict, source: str, source_name: str) -> dict: "realized_pnl": pos.get("realized_pnl_quote") or 0, "cum_fees": pos.get("cum_fees_quote") or 0, "executor_count": pos.get("executor_count") or 0, + "executor_ids": [str(i) for i in executor_ids], "source": source, "source_name": source_name, } diff --git a/condor/web/routes/reports.py b/condor/web/routes/reports.py index af3ed597b..98e01f35a 100644 --- a/condor/web/routes/reports.py +++ b/condor/web/routes/reports.py @@ -14,10 +14,42 @@ ) from condor.web.auth import get_current_user from condor.web.models import ReportsListResponse, ReportSummary, WebUser +from config_manager import get_config_manager router = APIRouter(prefix="/reports", tags=["reports"]) +def _owner_filter(user: WebUser) -> int | None: + """Whose reports a listing may show: everyone's for admins, own otherwise. + + ``None`` disables the store's owner filter — the admin override the other + server-data surfaces already grant (``_owner`` in conversations, + ``_require_ownership`` in sessions). Anyone else is scoped to entries + stamped with their own id; legacy entries with no owner are dropped for + them (fail closed, SEC-196). + """ + return None if get_config_manager().is_admin(user.id) else user.id + + +def _authorized_entry(report_id: str, user: WebUser) -> dict: + """Fetch an index entry, 404 if absent and 403 if it belongs to someone else. + + Ownership is the ``user_id`` the producer stamped at save time. Admins see + everything; an entry with no owner (written before SEC-196, or by a bare + script outside every runner) is admin-only rather than world-readable — + the same fail-closed rule ownerless routine instances follow. + """ + entry = get_report(report_id) + if not entry: + raise HTTPException(404, "Report not found") + if get_config_manager().is_admin(user.id): + return entry + owner = entry.get("user_id") + if owner is not None and owner == user.id: + return entry + raise HTTPException(status_code=403, detail="Not your report") + + @router.get("", response_model=ReportsListResponse) async def get_reports( source_type: str | None = None, @@ -35,6 +67,7 @@ async def get_reports( agent=agent, limit=limit, offset=offset, + owner_id=_owner_filter(user), ) return ReportsListResponse( reports=[ReportSummary(**e) for e in entries], @@ -44,28 +77,28 @@ async def get_reports( @router.get("/latest-by-source") async def get_reports_grouped(user: WebUser = Depends(get_current_user)): - return list_reports_grouped() + return list_reports_grouped(owner_id=_owner_filter(user)) @router.get("/{report_id}", response_model=ReportSummary) async def get_report_detail(report_id: str, user: WebUser = Depends(get_current_user)): - entry = get_report(report_id) - if not entry: - raise HTTPException(404, "Report not found") + entry = _authorized_entry(report_id, user) return ReportSummary(**entry) @router.get("/{report_id}/html", response_class=HTMLResponse) async def get_report_html(report_id: str, user: WebUser = Depends(get_current_user)): - """Return a report's rendered HTML body to an authenticated caller. + """Return a report's rendered HTML body to its owner (or an admin). Report bodies carry portfolio value, PnL, positions and strategy internals, so they are served from behind ``get_current_user`` like every other reports - surface — there is no unauthenticated static mount for them (SEC-112). The + surface — there is no unauthenticated static mount for them (SEC-112) — and, + since SEC-196, only to the report's recorded owner or an admin. The client fetches this with an ``Authorization`` header and hands the result to a sandboxed iframe via ``srcDoc``, which is why the body is returned inline rather than as a file the browser navigates to. """ + _authorized_entry(report_id, user) result = get_report_raw_html(report_id) if result is None: raise HTTPException(404, "Report not found") @@ -77,6 +110,7 @@ async def get_report_html(report_id: str, user: WebUser = Depends(get_current_us async def delete_report_endpoint( report_id: str, user: WebUser = Depends(get_current_user) ): + _authorized_entry(report_id, user) if not await delete_report(report_id): raise HTTPException(404, "Report not found") return {"deleted": True} diff --git a/condor/web/routes/servers.py b/condor/web/routes/servers.py index 9d94afd91..72d0645e1 100644 --- a/condor/web/routes/servers.py +++ b/condor/web/routes/servers.py @@ -47,6 +47,10 @@ async def list_servers(user: WebUser = Depends(get_current_user)): names = list(accessible) flags = await asyncio.gather(*(_resolve_online(sds, name) for name in names)) + # The selector seeds itself from this on a browser that has not picked a + # server yet, so a default set in Telegram is the one the dashboard opens on. + default_server = cm.get_chat_default_server(user.id) + results = [] for name, online in zip(names, flags): cfg = accessible[name] @@ -58,6 +62,7 @@ async def list_servers(user: WebUser = Depends(get_current_user)): port=cfg.get("port", 0), online=online, permission=perm.value if perm else "trader", + is_default=name == default_server, ) ) diff --git a/condor/web/routes/sessions.py b/condor/web/routes/sessions.py index d2a484605..deba0eac9 100644 --- a/condor/web/routes/sessions.py +++ b/condor/web/routes/sessions.py @@ -85,12 +85,12 @@ async def session_options(user: WebUser = Depends(get_current_user)): is a real key. """ from condor.agents.agent import AgentStore + from condor.llm.options import AGENT_OPTIONS, DEFAULT_AGENT from condor.preferences import ( get_active_agent_key, get_custom_providers, load_user_data_for, ) - from handlers.agents._shared import AGENT_OPTIONS, DEFAULT_AGENT cm = get_config_manager() providers = get_custom_providers(load_user_data_for(user.id)) diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index d996f5610..e97ba4fb9 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -2,6 +2,8 @@ import asyncio import logging +import re +from urllib.parse import urlsplit from fastapi import APIRouter, Depends, HTTPException, Query @@ -85,6 +87,11 @@ async def _get_status(name: str) -> dict: statuses = await asyncio.gather(*[_get_status(name) for name in accessible]) + # Which one the star should be filled on. Read once: the config manager falls + # back to the global default and then to the first server, so this is the + # server that actually answers, not only an explicit `chat_defaults` entry. + default_server = cm.get_chat_default_server(user.id) + results = [] for (name, cfg), status in zip(accessible.items(), statuses): perm = cm.get_server_permission(user.id, name) @@ -96,6 +103,7 @@ async def _get_status(name: str) -> dict: port=cfg.get("port", 0), online=online, permission=perm.value if perm else "trader", + is_default=name == default_server, ) ) @@ -158,6 +166,13 @@ async def set_default_server(name: str, user: WebUser = Depends(get_current_user if not cm.has_server_access(user.id, name): raise HTTPException(status_code=404, detail="Server not found") cm.set_chat_default_server(user.id, name) + # Both records Telegram's own "set default" writes, or the two disagree: + # `chat_defaults` is what `get_effective_server` resolves, while the general + # preference is what a routine, an agent or MCP reads as the active server. + # Writing only the first left those surfaces pinned to the previous choice. + from condor.preferences import load_user_data_for, set_active_server + + set_active_server(load_user_data_for(user.id), name) return {"default": True, "name": name} @@ -300,6 +315,46 @@ async def gateway_logs( # ── Gateway Networks (RPC) ── +_URL_VALUE = re.compile(r"^[a-z][a-z0-9+.\-]*://", re.IGNORECASE) + + +def _redact_url(value: str) -> str: + """Collapse a URL to scheme://host, hiding every part that can carry a secret. + + Paid RPC providers embed the API key either in the query string + (Helius: ``?api-key=``) or in the path (Infura ``/v3/``, + Alchemy ``/v2/``), and a URL can also carry userinfo. Rather than + pattern-match key shapes, keep only the parts that cannot hold a + credential — scheme, host, port — and mask everything else whenever any of + it is present. A bare endpoint stays readable so a trader can still tell + which node they are trading against. + """ + parts = urlsplit(value) + host = parts.hostname or "" + if parts.port: + host = f"{host}:{parts.port}" + has_secret_bearing_part = ( + parts.username + or parts.password + or parts.query + or parts.fragment + or parts.path not in ("", "/") + ) + if has_secret_bearing_part: + return f"{parts.scheme}://{host}/…" + return value + + +def _redact_network_config(value): + """Recursively mask URL-shaped strings in a network config (SEC-197).""" + if isinstance(value, dict): + return {k: _redact_network_config(v) for k, v in value.items()} + if isinstance(value, list): + return [_redact_network_config(v) for v in value] + if isinstance(value, str) and _URL_VALUE.match(value): + return _redact_url(value) + return value + @router.get("/gateway/networks") async def gateway_networks( @@ -326,10 +381,20 @@ async def gateway_network_config( user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() + # Reading gateway state stays at TRADER (see _require_owner's docstring), + # but the RPC URL in a network config embeds the owner's paid provider API + # key. Redact it server-side for anyone below the OWNER line — same admin + # bypass as _require_owner — instead of shipping the secret and hiding it + # in the UI (SEC-197). Redaction cannot corrupt a save: the POST below is + # owner-gated, and owners always receive the full values. + perm = cm.get_server_permission(user.id, server) + is_owner = perm == ServerPermission.OWNER or cm.is_admin(user.id) client = await _get_client(cm, server) try: config = await client.gateway.get_network_config(network_id) - return {"network_id": network_id, "config": config} + if not is_owner: + config = _redact_network_config(config) + return {"network_id": network_id, "config": config, "redacted": not is_owner} except Exception as e: logger.exception( "Failed to fetch gateway network config '%s' from '%s'", network_id, server @@ -742,17 +807,17 @@ async def add_custom_provider( the key is accepted, and the response is OpenAI-shaped — all in one round trip — and returns the model list the caller needs anyway. """ + from condor.llm.custom_models import ( + CustomProviderError, + fetch_models, + normalize_base_url, + ) from condor.preferences import ( load_user_data_for, save_custom_provider, suggest_provider_name, unique_provider_name, ) - from handlers.agents.custom_models import ( - CustomProviderError, - fetch_models, - normalize_base_url, - ) raw_url = (body.get("base_url") or "").strip() api_key = (body.get("api_key") or "").strip() @@ -789,8 +854,8 @@ async def get_custom_provider_models( user: WebUser = Depends(get_current_user), ): """Fetch the current chat model list for a saved endpoint.""" + from condor.llm.custom_models import CustomProviderError, fetch_models from condor.preferences import find_custom_provider, load_user_data_for - from handlers.agents.custom_models import CustomProviderError, fetch_models provider = find_custom_provider(load_user_data_for(user.id), name) if provider is None: @@ -826,13 +891,20 @@ async def delete_custom_provider( async def get_telemetry_settings(user: WebUser = Depends(get_current_user)): """What this install has agreed to, and whether it could send anything. - ``endpoint_configured`` is the honest answer to "is this thing on": with no - ``CONDOR_TELEMETRY_URL`` set — the shipped state — nothing can be - transmitted no matter what the consent says, and events only accumulate in - a capped local file. + The collector address is fixed in the source, so consent is the only thing + that decides whether anything is transmitted: at level ``off`` no event is + recorded in the first place. + + Readable by every seat on purpose — anyone using an install should be able + to see what it shares — while ``can_change`` reports who may actually + answer. That is the admin, because consent is install-wide, and it is also + the discriminator the dashboard consent card uses, so asking costs no + second request. ``disclosure`` is the same copy the Telegram prompt sends. """ from condor.telemetry import consent, emitter, outbox + from condor.telemetry.prompt import DISCLOSURE + cm = get_config_manager() return { "consent": consent.state(), "level": consent.level(), @@ -840,18 +912,22 @@ async def get_telemetry_settings(user: WebUser = Depends(get_current_user)): "endpoint_configured": bool(outbox.endpoint()), "pending_events": emitter.buffered(), "privacy_doc": "PRIVACY.md", + "can_change": cm.is_admin(user.id), + "disclosure": DISCLOSURE, } @router.put("/telemetry") async def set_telemetry_settings( - level: str = Query(..., description="off | ping | usage"), + level: str = Query(..., description="ping | usage"), user: WebUser = Depends(get_current_user), ): """Change the install's telemetry level. Admin only, and reversible. - Setting ``off`` is a withdrawal, not a pause: the buffer and the outbox are - deleted, so nothing already recorded can be sent later. + ``ping`` is the floor — install counting cannot be turned off here; only + ``CONDOR_TELEMETRY=off`` in the environment silences telemetry entirely. + Downgrading from ``usage`` is a withdrawal, not a pause: the buffer and the + outbox are deleted, so nothing already recorded can be sent later. """ from condor.telemetry import consent @@ -861,9 +937,12 @@ async def set_telemetry_settings( status_code=403, detail="Telemetry is an install-wide setting; only the admin can change it", ) - if level not in consent.LEVELS: + if level not in (consent.PING, consent.USAGE): raise HTTPException( - status_code=400, detail=f"level must be one of {', '.join(consent.LEVELS)}" + status_code=400, + detail="level must be ping or usage; install counting cannot be " + "turned off here — set CONDOR_TELEMETRY=off in the environment to " + "disable telemetry entirely", ) if consent.env_overridden(): raise HTTPException( diff --git a/condor/web/streams/__init__.py b/condor/web/streams/__init__.py new file mode 100644 index 000000000..1c3b53512 --- /dev/null +++ b/condor/web/streams/__init__.py @@ -0,0 +1,47 @@ +"""Upstream stream implementations behind the ``WebSocketManager``. + +Each module in this package owns one stream family: + +- ``candles``: the candle subsystem — per-channel buffer, backend WS stream, + REST poll fallback, historical backfill and idle-buffer cleanup. +- ``hummingbot_ws``: the streams bridged from the Hummingbot backend WS + endpoints (trades, order book, executors, bots, positions, performance) + plus the controller-performance REST poll. + +The stream code is packaged as mixins over a small host surface +(:class:`StreamHost`) that ``WebSocketManager`` implements: broadcasting, +subscriber checks, per-connection sends and the shared one-shot task tracker. +A unit test can drive a mixin with a stub host instead of the full manager, +and ``WebSocketManager`` itself keeps only connection bookkeeping, the SDS +bridge and the stream lifecycle registry. +""" + +from __future__ import annotations + +from typing import Any, Protocol + +from condor.asyncutil import TaskSet + + +class StreamHost(Protocol): + """What the stream mixins require of the class they are mixed into. + + ``WebSocketManager`` is the one production implementation; the protocol + exists so each stream family can be unit-tested against a stub. + """ + + _last_data: dict[str, Any] + _oneshot_tasks: TaskSet + + async def broadcast(self, channel: str, data: Any) -> None: ... + + async def _broadcast_update(self, channel: str, data: Any) -> None: ... + + async def _send(self, conn: Any, channel: str, data: Any) -> None: ... + + def _has_subscribers(self, channel: str) -> bool: ... + + def _ensure_stream(self, prefix: str, channel: str) -> None: ... + + +__all__ = ["StreamHost"] diff --git a/condor/web/streams/candles.py b/condor/web/streams/candles.py new file mode 100644 index 000000000..ac6534d86 --- /dev/null +++ b/condor/web/streams/candles.py @@ -0,0 +1,720 @@ +"""Candle streaming subsystem: buffer, backend WS stream, REST poll fallback, +historical backfill and idle-buffer cleanup. + +``CandleStreamsMixin`` is mixed into ``WebSocketManager`` (the +:class:`~condor.web.streams.StreamHost` implementation): it calls the host's +``broadcast`` / ``_send`` / ``_has_subscribers`` / ``_ensure_stream`` and +tracks its fire-and-forget work on the host's ``_oneshot_tasks``. The candle +state it drives (``_candle_buffers``, ``_candle_tasks``, +``_candle_poll_tasks``, teardown timers and staleness bookkeeping) is +initialized by the host's ``__init__``. + +Only this module touches ``_CandleBuffer`` internals; everything else goes +through its accessors. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import math +import time +from typing import TYPE_CHECKING, Any + +from condor import dex_candles +from condor.asyncutil import TaskSet +from condor.fetchers.market_data import fetch_historical_candles, normalize_candle + +if TYPE_CHECKING: + from condor.web.ws_manager import _Connection + +logger = logging.getLogger(__name__) + +# Interval string -> seconds for buffer sizing +_INTERVAL_SECONDS: dict[str, int] = { + "1s": 1, + "5s": 5, + "15s": 15, + "30s": 30, + "1m": 60, + "3m": 180, + "5m": 300, + "15m": 900, + "30m": 1800, + "1h": 3600, + "2h": 7200, + "4h": 14400, + "1d": 86400, + "1w": 604800, +} + +# Auto-cleanup candle buffers unused for this long +_CANDLE_BUFFER_IDLE_TTL = 600 # 10 minutes + + +def _coerce_duration(value: object) -> int | None: + """Coerce a client-supplied candle duration to a positive int (seconds). + + WS messages are untrusted input: a non-numeric value must not raise, or the + ValueError would escape handle_message and tear down the client's entire + multiplexed connection. Returns None for missing/invalid/non-positive values. + """ + if value is None or isinstance(value, bool): + return None + try: + duration = int(float(value)) + except (TypeError, ValueError): + logger.warning("Ignoring invalid candle duration from client: %r", value) + return None + return duration if duration > 0 else None + + +def parse_candle_channel(channel: str) -> tuple[str, str, str, str, str | None] | None: + """Split a candle channel into its parts, or ``None`` if it is malformed. + + The channel is ``candles:{server}:{connector}:{pair}:{interval}[:{pool_address}]``. + The pool address is an **optional sixth segment**, set by the DEX page so a + chart is pinned to the exact pool the user opened instead of whichever pool + ``dex_candles._resolve_pool`` judges deepest for the pair. A five-part channel + (every CLOB chart) parses to ``pool_address=None`` and is byte-identical to + what it was before the segment existed. + + Every consumer must go through here: unpacking five names off ``split(":")`` + raises ``ValueError`` on a six-part channel, and each of the three call sites + fails silently in a different way (a stream that never backfills, a backfill + that never ticks). + """ + parts = channel.split(":") + if len(parts) < 5: + return None + server_name, connector, pair, interval = parts[1:5] + pool_address = parts[5] if len(parts) > 5 else None + return server_name, connector, pair, interval, pool_address or None + + +class _CandleBuffer: + """Per-channel candle buffer with dynamic sizing based on interval + duration.""" + + __slots__ = ("interval", "_data", "_max_size", "last_accessed") + + def __init__(self, interval: str, duration_seconds: int = 3600): + self.interval = interval + self._data: dict[float, dict] = {} + self._max_size: int = 200 + self.last_accessed: float = time.monotonic() + self.set_duration(duration_seconds) + + def set_duration(self, duration_seconds: int) -> int: + """Resize buffer for the given duration. Returns the new max size.""" + interval_sec = _INTERVAL_SECONDS.get(self.interval, 60) + needed = math.ceil(duration_seconds / interval_sec) + new_max = max(needed, 200) # minimum 200 + old_max = self._max_size + self._max_size = new_max + self.last_accessed = time.monotonic() + self._evict() + if new_max != old_max: + logger.debug( + "Candle buffer resized %s: %d -> %d (duration=%ds)", + self.interval, + old_max, + new_max, + duration_seconds, + ) + return new_max + + def upsert(self, candle: dict) -> None: + self._data[candle["timestamp"]] = candle + self._evict() + + def upsert_many(self, candles: list[dict]) -> None: + for c in candles: + self._data[c["timestamp"]] = c + self._evict() + + def get_sorted(self) -> list[dict]: + self.last_accessed = time.monotonic() + return sorted(self._data.values(), key=lambda c: c["timestamp"]) + + def get(self, timestamp: float) -> dict | None: + """The buffered candle at ``timestamp``, or ``None``.""" + return self._data.get(timestamp) + + def _evict(self) -> None: + excess = len(self._data) - self._max_size + if excess <= 0: + return + # Drop the `excess` oldest timestamps in a single O(n log n) pass + # instead of calling min() (O(n)) per evicted candle. + for ts in sorted(self._data)[:excess]: + del self._data[ts] + + @property + def size(self) -> int: + return len(self._data) + + @property + def max_size(self) -> int: + return self._max_size + + @property + def latest_timestamp(self) -> float: + """Timestamp of the newest buffered candle, or 0 when empty.""" + return max(self._data) if self._data else 0 + + @property + def needs_backfill(self) -> bool: + """True if buffer has room for significantly more candles.""" + return len(self._data) < self._max_size * 0.5 + + +class CandleStreamsMixin: + """Candle stream family: subscribe handling, backfill, backend WS stream, + REST poll fallback, deferred teardown and idle-buffer cleanup.""" + + _CANDLE_KEEP_ALIVE = 300 # 5-minute grace period before tearing down candle streams + + if TYPE_CHECKING: + # Candle state, initialized by WebSocketManager.__init__. + _candle_tasks: dict[str, asyncio.Task] + _candle_poll_tasks: dict[str, asyncio.Task] + _candle_buffers: dict[str, _CandleBuffer] + _candle_teardown_timers: dict[str, asyncio.TimerHandle] + _last_candle_ws_update: dict[str, float] + _candle_first_msg_logged: set[str] + _connections: list[Any] + # Host surface (see condor.web.streams.StreamHost). + _last_data: dict[str, Any] + _oneshot_tasks: TaskSet + + async def broadcast(self, channel: str, data: Any) -> None: ... + + async def _send(self, conn: Any, channel: str, data: Any) -> None: ... + + def _has_subscribers(self, channel: str) -> bool: ... + + def _ensure_stream(self, prefix: str, channel: str) -> None: ... + + # Provided by HummingbotStreamsMixin on the composed manager. + @staticmethod + def _is_permanent_ws_error(error_str: str) -> bool: ... + + @staticmethod + def _normalize_candle(c: Any) -> dict | None: + """Normalize a candle from any format to a uniform dict with float values.""" + return normalize_candle(c) + + # -- Subscribe handling -- + + async def _handle_candle_subscribe( + self, conn: "_Connection", channel: str, duration: object + ) -> None: + """Handle candle channel subscription with optional duration.""" + parts = channel.split(":") + if len(parts) < 5: + return + interval = parts[4] + dur = _coerce_duration(duration) or 3 * 86400 # default 3 days + + # Cancel any pending teardown — a subscriber just came back + timer = self._candle_teardown_timers.pop(channel, None) + if timer is not None: + timer.cancel() + logger.debug("Cancelled pending candle teardown for %s", channel) + + buf = self._candle_buffers.get(channel) + if buf is None: + buf = _CandleBuffer(interval, dur) + self._candle_buffers[channel] = buf + else: + # Expand buffer if this client needs more + if dur > 0: + old_max = buf.max_size + buf.set_duration(dur) + if buf.max_size > old_max and buf.needs_backfill: + self._oneshot_tasks.track( + asyncio.create_task( + self._backfill_candles(channel), + name=f"backfill:{channel}", + ) + ) + + # Send buffered candles as initial snapshot + sorted_candles = buf.get_sorted() + if sorted_candles: + await self._send(conn, channel, {"type": "candles", "data": sorted_candles}) + + # If the stream task is still running (kept alive during grace period), skip restart + self._ensure_stream("candles", channel) + + async def _handle_candle_duration_change( + self, conn: "_Connection", channel: str, duration: int + ) -> None: + """Handle duration change for an existing candle subscription. + + Only grows the buffer — never shrinks. The frontend manages its own + display window; the backend just ensures enough history is buffered. + """ + buf = self._candle_buffers.get(channel) + if buf is None: + return + old_max = buf.max_size + # Only expand — skip if requested duration would shrink the buffer + interval_sec = _INTERVAL_SECONDS.get(buf.interval, 60) + needed = max(math.ceil(duration / interval_sec), 200) + if needed <= old_max: + return + buf.set_duration(duration) + if buf.needs_backfill: + await self._backfill_candles(channel) + # Broadcast updated snapshot to ALL subscribers on this channel + sorted_candles = buf.get_sorted() + if sorted_candles: + await self.broadcast(channel, {"type": "candles", "data": sorted_candles}) + + async def _backfill_candles(self, channel: str) -> None: + """Fetch historical candles to fill the buffer gap.""" + parsed = parse_candle_channel(channel) + if parsed is None: + return + server_name, connector, pair, interval, pool_address = parsed + buf = self._candle_buffers.get(channel) + if buf is None: + return + + from config_manager import get_config_manager + + cm = get_config_manager() + try: + interval_sec = _INTERVAL_SECONDS.get(interval, 60) + end_time = int(time.time()) + start_time = end_time - (buf.max_size * interval_sec) + + logger.info( + "Backfilling candles for %s: need %d, have %d, fetching %ds-%ds", + channel, + buf.max_size, + buf.size, + start_time, + end_time, + ) + + if dex_candles.uses_gecko_candles(connector): + candles = await dex_candles.fetch_dex_candles( + connector, pool_address, pair, interval, start_time, end_time + ) + if candles: + buf.upsert_many(candles) + logger.info( + "Backfilled %d GeckoTerminal candles for %s (buffer: %d/%d)", + len(candles), + channel, + buf.size, + buf.max_size, + ) + return + + client = await cm.get_client(server_name) + candles = await fetch_historical_candles( + client, + connector, + pair, + interval, + start_time=start_time, + end_time=end_time, + limit=min(buf.max_size, 5000), + ) + if candles: + buf.upsert_many(candles) + logger.info( + "Backfilled %d candles for %s (buffer: %d/%d)", + len(candles), + channel, + buf.size, + buf.max_size, + ) + except Exception as e: + logger.warning("Candle backfill failed for %s: %s", channel, e) + + # -- Deferred teardown -- + + def _maybe_stop_candle_stream(self, channel: str) -> None: + # If subscribers still exist, cancel any pending teardown and return + if self._has_subscribers(channel): + timer = self._candle_teardown_timers.pop(channel, None) + if timer is not None: + timer.cancel() + return + + # Already have a pending teardown scheduled — nothing to do + if channel in self._candle_teardown_timers: + return + + # Schedule deferred teardown + loop = asyncio.get_event_loop() + handle = loop.call_later( + self._CANDLE_KEEP_ALIVE, + self._deferred_stop_candle_stream, + channel, + ) + self._candle_teardown_timers[channel] = handle + logger.info( + "Scheduled candle stream teardown for %s in %ds", + channel, + self._CANDLE_KEEP_ALIVE, + ) + + def _deferred_stop_candle_stream(self, channel: str) -> None: + """Actually tear down a candle stream after the grace period.""" + self._candle_teardown_timers.pop(channel, None) + + # Re-check: subscribers may have appeared during the grace period + if self._has_subscribers(channel): + logger.debug( + "Candle teardown cancelled — subscribers returned for %s", channel + ) + return + + task = self._candle_tasks.pop(channel, None) + if task and not task.done(): + task.cancel() + logger.info("Deferred candle stream teardown completed for %s", channel) + poll_task = self._candle_poll_tasks.pop(channel, None) + if poll_task and not poll_task.done(): + poll_task.cancel() + self._last_candle_ws_update.pop(channel, None) + self._candle_first_msg_logged.discard(channel) + # Snapshot-on-resubscribe is served by _candle_buffers, not _last_data + self._last_data.pop(channel, None) + # NOTE: candle buffer is NOT deleted — the existing idle cleanup loop handles that + + # -- Backend WS stream -- + + async def _candle_stream(self, channel: str) -> None: + parsed = parse_candle_channel(channel) + if parsed is None: + return + server_name, connector, pair, interval, pool_address = parsed + + from config_manager import get_config_manager + + cm = get_config_manager() + backoff = 5 + + # Start REST poll fallback alongside the WS stream + self._ensure_candle_poll_fallback(channel) + + if dex_candles.uses_gecko_candles(connector): + # No CandlesFactory feed to subscribe to (xrpl, DEX networks): the + # poll loop above is the whole stream. Subscribing anyway would just + # reconnect-loop on an error the API can never stop returning. + logger.info( + "Candle stream for %s is GeckoTerminal-polled (no API feed)", channel + ) + return + + while True: + try: + client = await cm.get_client(server_name) + async with client.ws.market_data() as ws: + await ws.subscribe_candles( + connector, + pair, + interval=interval, + max_records=100, + update_interval=1.0, + ) + logger.info("Candle WS subscribed: %s", channel) + backoff = 5 # Reset on successful connection + async for msg in ws: + # Log first message per channel at INFO for diagnostics + if channel not in self._candle_first_msg_logged: + self._candle_first_msg_logged.add(channel) + logger.info( + "Candle WS first message for %s: %s", + channel, + ( + json.dumps(msg)[:500] + if isinstance(msg, dict) + else type(msg).__name__ + ), + ) + else: + logger.debug( + "Candle WS raw msg keys for %s: %s", + channel, + ( + list(msg.keys()) + if isinstance(msg, dict) + else type(msg).__name__ + ), + ) + + msg_type = msg.get("type") + if msg_type == "candle_update": + raw = msg.get("data") + candle = self._normalize_candle(raw) if raw else None + if candle: + self._last_candle_ws_update[channel] = time.monotonic() + self._upsert_candle_buffer(channel, candle) + await self.broadcast( + channel, + {"type": "candle_update", "candle": candle}, + ) + elif msg_type == "candles": + raw_list = msg.get("data") or [] + candles = [ + c + for r in raw_list + if (c := self._normalize_candle(r)) is not None + ] + if candles: + self._last_candle_ws_update[channel] = time.monotonic() + self._upsert_candle_buffer_many(channel, candles) + await self.broadcast( + channel, + {"type": "candles", "data": candles}, + ) + elif msg_type in ("heartbeat", "subscribed"): + continue + elif msg_type == "error": + error_msg = msg.get("message", "unknown error") + logger.warning( + "Candle stream error for %s: %s — continuing", + channel, + error_msg, + ) + await self.broadcast( + channel, + { + "type": "error", + "message": f"Stream error: {error_msg}", + }, + ) + # Don't break — the WS may still be alive. + # If truly dead, next recv raises and we reconnect. + continue + else: + logger.info( + "Candle stream unrecognized msg type for %s: type=%s keys=%s", + channel, + msg_type, + ( + list(msg.keys()) + if isinstance(msg, dict) + else type(msg).__name__ + ), + ) + + except asyncio.CancelledError: + return + except Exception as e: + error_str = str(e) + # Detect permanent failures — don't retry (issue #134, see + # _is_permanent_ws_error). + if self._is_permanent_ws_error(error_str): + logger.warning( + "Candle stream permanent error for %s: %s — giving up", + channel, + e, + ) + await self.broadcast( + channel, + {"type": "error", "message": f"Stream failed: {error_str}"}, + ) + # Stop the paired REST poll fallback — nothing valid to poll. + poll_task = self._candle_poll_tasks.pop(channel, None) + if poll_task and not poll_task.done(): + poll_task.cancel() + return + + logger.warning( + "Candle stream error for %s: %s, reconnecting in %ds...", + channel, + e, + backoff, + ) + await self.broadcast( + channel, + { + "type": "error", + "message": f"Connection lost, retrying in {backoff}s", + }, + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 60) + + # -- REST poll fallback -- + + def _ensure_candle_poll_fallback(self, channel: str) -> None: + """Start a REST poll fallback task for a candle channel if not already running.""" + if ( + channel in self._candle_poll_tasks + and not self._candle_poll_tasks[channel].done() + ): + return + self._candle_poll_tasks[channel] = asyncio.create_task( + self._candle_poll_fallback(channel) + ) + + async def _candle_poll_fallback(self, channel: str) -> None: + """Periodically poll REST for candles when WS stream goes silent. + + For a GeckoTerminal-backed connector there is no WS stream to fall back + from — this loop *is* the feed, so it polls from the first tick and never + waits for staleness. + """ + parsed = parse_candle_channel(channel) + if parsed is None: + return + server_name, connector, pair, interval, pool_address = parsed + interval_sec = _INTERVAL_SECONDS.get(interval, 60) + gecko = dex_candles.uses_gecko_candles(connector) + # How long without a WS update before we consider it stale + stale_threshold = max(interval_sec, 15) + # How often to poll REST once stale (keep the last candle fresh). Gecko + # polls stay well clear of its per-minute rate limit — every chart on the + # dashboard shares one budget — while still refreshing a forming candle. + poll_interval = ( + max(min(interval_sec // 2, 60), 30) if gecko else min(interval_sec, 10) + ) + was_stale = gecko + + from config_manager import get_config_manager + + try: + while True: + await asyncio.sleep(poll_interval if was_stale else stale_threshold) + + last_update = self._last_candle_ws_update.get(channel) + if ( + last_update is not None + and (time.monotonic() - last_update) <= stale_threshold + ): + if was_stale: + logger.info( + "Candle WS stream resumed for %s, stopping REST fallback polling", + channel, + ) + was_stale = False + continue + + # Stream is stale — poll REST + if not was_stale: + logger.warning( + "Candle WS stream appears stale for %s (threshold=%ds), polling REST fallback", + channel, + stale_threshold, + ) + was_stale = True + + try: + now = int(time.time()) + if gecko: + candles = await dex_candles.fetch_dex_candles( + connector, + pool_address, + pair, + interval, + now - interval_sec * 5, + now, + # The shared OHLCV cache holds a pool for minutes, + # long enough to freeze a live chart. + use_cache=False, + ) + else: + cm = get_config_manager() + client = await cm.get_client(server_name) + # No `limit`: this poll only wants the fresh tail of the + # live range, so an empty historical answer stays empty + # rather than pulling a full unrelated window. + candles = await fetch_historical_candles( + client, + connector, + pair, + interval, + start_time=now - interval_sec * 5, + end_time=now, + ) + if candles: + buf = self._candle_buffers.get(channel) + # Broadcast if we have newer candles OR if the latest + # candle's OHLCV changed (same timestamp, updated values) + newest_buf_ts = buf.latest_timestamp if buf else 0 + newest_poll_ts = max(c["timestamp"] for c in candles) + changed = newest_poll_ts > newest_buf_ts + if not changed and buf: + old = buf.get(newest_poll_ts) + if old is not None: + # Same timestamp — check if OHLCV actually changed + new = next( + c + for c in candles + if c["timestamp"] == newest_poll_ts + ) + changed = any( + old.get(k) != new.get(k) + for k in ("open", "high", "low", "close", "volume") + ) + if changed: + self._upsert_candle_buffer_many(channel, candles) + await self.broadcast( + channel, + {"type": "candles", "data": candles}, + ) + logger.debug( + "REST fallback delivered %d candles for %s", + len(candles), + channel, + ) + except Exception as e: + logger.debug("REST candle poll failed for %s: %s", channel, e) + + except asyncio.CancelledError: + return + + # -- Buffer helpers -- + + def _get_or_create_candle_buffer(self, channel: str) -> _CandleBuffer: + """Get existing buffer or create with default duration.""" + buf = self._candle_buffers.get(channel) + if buf is None: + parts = channel.split(":") + interval = parts[4] if len(parts) >= 5 else "1m" + buf = _CandleBuffer(interval, 3 * 86400) # default 3 days + self._candle_buffers[channel] = buf + return buf + + def _upsert_candle_buffer(self, channel: str, candle: dict) -> None: + """Upsert a single candle into the per-channel buffer.""" + buf = self._get_or_create_candle_buffer(channel) + buf.upsert(candle) + + def _upsert_candle_buffer_many(self, channel: str, candles: list[dict]) -> None: + """Upsert multiple candles into the per-channel buffer.""" + buf = self._get_or_create_candle_buffer(channel) + buf.upsert_many(candles) + + async def _candle_buffer_cleanup_loop(self) -> None: + """Periodically remove candle buffers that haven't been accessed.""" + try: + while True: + await asyncio.sleep(60) # check every minute + now = time.monotonic() + stale = [ + ch + for ch, buf in self._candle_buffers.items() + if (now - buf.last_accessed) > _CANDLE_BUFFER_IDLE_TTL + and not any(ch in c.channels for c in self._connections) + ] + for ch in stale: + buf = self._candle_buffers.pop(ch, None) + if buf: + logger.info( + "Cleaned up idle candle buffer: %s (%d candles)", + ch, + buf.size, + ) + except asyncio.CancelledError: + return diff --git a/condor/web/streams/hummingbot_ws.py b/condor/web/streams/hummingbot_ws.py new file mode 100644 index 000000000..1056505e0 --- /dev/null +++ b/condor/web/streams/hummingbot_ws.py @@ -0,0 +1,599 @@ +"""Streams bridged from the Hummingbot backend WS endpoints, plus the +controller-performance REST poll. + +``HummingbotStreamsMixin`` is mixed into ``WebSocketManager`` (the +:class:`~condor.web.streams.StreamHost` implementation). It owns the shared +connect/reconnect skeleton (``_run_ws_stream``) and every per-protocol +handler built on it: trades and order book (``/ws/market-data``), executors, +bots, positions and performance (``/ws/executors``), and the 30s +controller-performance poll. Channel names and message shapes are part of the +dashboard's wire contract — the frontend depends on them byte-for-byte. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING, Any, Awaitable, Callable + +logger = logging.getLogger(__name__) + + +class HummingbotStreamsMixin: + """Upstream Hummingbot WS stream family and its shared reconnect skeleton.""" + + if TYPE_CHECKING: + # Stream task state, initialized by WebSocketManager.__init__. + _controller_perf_tasks: dict[str, asyncio.Task] + # Host surface (see condor.web.streams.StreamHost). + _last_data: dict[str, Any] + + async def broadcast(self, channel: str, data: Any) -> None: ... + + async def _broadcast_update(self, channel: str, data: Any) -> None: ... + + def _has_subscribers(self, channel: str) -> bool: ... + + # -- Transforms (REST-shape parity for WS broadcasts) -- + + @staticmethod + def _transform_executors(raw_data: Any) -> list[dict]: + """Transform raw executor data to ExecutorInfo-compatible dicts for WS broadcast.""" + from condor.fetchers.executors import extract_executors_list + from condor.web.models import ExecutorInfo + + executors_list = extract_executors_list(raw_data) + result = [] + for ex in executors_list: + info = ExecutorInfo.from_raw(ex) + if info: + result.append(info.model_dump()) + return result + + @staticmethod + def _transform_bots(raw_data: Any) -> dict: + """Transform raw BOTS_STATUS data to BotsPageResponse-compatible dict for WS broadcast.""" + from condor.fetchers.bots import build_bots_page + + return build_bots_page(raw_data) + + @staticmethod + def _overlay_stopping_state(server_name: str, data: dict) -> None: + """Apply transitional 'stopping' state to WS broadcast data.""" + from condor.web.routes.bots import overlay_stopping_state + + overlay_stopping_state( + server_name, data.get("controllers", []), data.get("bots", []) + ) + + # -- Shared skeleton -- + + @staticmethod + def _is_permanent_ws_error(error_str: str) -> bool: + """True if a stream error will never succeed on retry. + + Besides auth/not-found (401/403/404), an invalid trading pair (wrong + symbol for this connector) will never become valid, so retrying + forever just spams logs and hammers the exchange until Condor + restarts (issue #134). + """ + lowered = error_str.lower() + return ( + any(code in error_str for code in ("401", "403", "404")) + or "appears to be invalid" in lowered + or "invalid symbol" in lowered + ) + + async def _run_ws_stream( + self, + channel: str, + server_name: str, + *, + label: str, + open_ws: Callable[[Any], Any], + subscribe: Callable[[Any], Awaitable[None]], + on_message: Callable[[dict], Awaitable[None]], + backoff_on_empty_close: bool = False, + ) -> None: + """Shared connect/reconnect skeleton for all Hummingbot WS streams. + + Owns the while-True loop, subscriber check, heartbeat/error message + handling, permanent-error detection (``_is_permanent_ws_error``) and + exponential backoff capped at 60s. Each stream supplies only: + + - ``open_ws``: client -> the WS async context manager to enter + (e.g. ``lambda c: c.ws.market_data()``). + - ``subscribe``: performs the subscription on the open socket. + - ``on_message``: handles data messages (heartbeat/error are + handled here). + + ``backoff_on_empty_close`` reproduces the executor stream's + behavior: don't reset backoff on subscribe; after a clean close, + reset it only if at least one message arrived, otherwise back off + (guards against a server that accepts then immediately drops). + """ + from config_manager import get_config_manager + + cm = get_config_manager() + backoff = 5 + lower_label = label[0].lower() + label[1:] + subscribed_label = label if label.endswith("WS") else f"{label} WS" + + while True: + try: + client = await cm.get_client(server_name) + async with open_ws(client) as ws: + await subscribe(ws) + logger.info("%s subscribed: %s", subscribed_label, channel) + if not backoff_on_empty_close: + backoff = 5 + got_message = False + async for msg in ws: + if not self._has_subscribers(channel): + logger.info( + "No subscribers for %s, closing %s stream", + channel, + lower_label, + ) + return + + got_message = True + msg_type = msg.get("type") + if msg_type == "heartbeat": + continue + if msg_type == "error": + logger.warning( + "%s stream error for %s: %s", + label, + channel, + msg.get("message", "unknown error"), + ) + break + await on_message(msg) + + if backoff_on_empty_close: + # Connection closed cleanly — back off if short-lived + if got_message: + backoff = 5 + else: + logger.warning( + "%s stream closed immediately for %s, " + "reconnecting in %ds...", + label, + channel, + backoff, + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 60) + + except asyncio.CancelledError: + return + except Exception as e: + if self._is_permanent_ws_error(str(e)): + logger.warning( + "%s stream permanent error for %s: %s — giving up", + label, + channel, + e, + ) + return + + logger.warning( + "%s stream error for %s: %s, reconnecting in %ds...", + label, + channel, + e, + backoff, + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 60) + + # -- Trade streaming -- + + async def _trade_stream(self, channel: str) -> None: + parts = channel.split(":") + if len(parts) < 4: + return + _, server_name, connector, pair = parts[:4] + + async def subscribe(ws: Any) -> None: + await ws.subscribe_trades( + connector, + pair, + update_interval=1.0, + ) + + async def on_message(msg: dict) -> None: + if msg.get("type") != "trades": + return + trade_data = msg.get("data", []) + trades = [] + for t in trade_data: + if isinstance(t, dict): + trades.append( + { + "price": float(t.get("price", 0)), + "amount": float(t.get("amount", t.get("quantity", 0))), + "side": t.get("side", t.get("trade_type", "buy")).lower(), + "timestamp": float(t.get("timestamp", 0)), + } + ) + if trades: + await self.broadcast(channel, {"type": "trades", "data": trades}) + + await self._run_ws_stream( + channel, + server_name, + label="Trade", + open_ws=lambda client: client.ws.market_data(), + subscribe=subscribe, + on_message=on_message, + ) + + # -- Order book streaming -- + + async def _order_book_stream(self, channel: str) -> None: + parts = channel.split(":") + if len(parts) < 4: + return + _, server_name, connector, pair = parts[:4] + + async def subscribe(ws: Any) -> None: + await ws.subscribe_order_book( + connector, + pair, + depth=20, + update_interval=1.0, + ) + + async def on_message(msg: dict) -> None: + if msg.get("type") != "order_book": + return + raw_data = msg.get("data", {}) + bids = [] + asks = [] + for b in raw_data.get("bids") or []: + if isinstance(b, dict): + bids.append( + { + "price": float(b.get("price", 0)), + "amount": float(b.get("amount", b.get("quantity", 0))), + } + ) + elif isinstance(b, (list, tuple)) and len(b) >= 2: + bids.append({"price": float(b[0]), "amount": float(b[1])}) + for a in raw_data.get("asks") or []: + if isinstance(a, dict): + asks.append( + { + "price": float(a.get("price", 0)), + "amount": float(a.get("amount", a.get("quantity", 0))), + } + ) + elif isinstance(a, (list, tuple)) and len(a) >= 2: + asks.append({"price": float(a[0]), "amount": float(a[1])}) + await self.broadcast(channel, {"bids": bids, "asks": asks}) + + await self._run_ws_stream( + channel, + server_name, + label="Order book", + open_ws=lambda client: client.ws.market_data(), + subscribe=subscribe, + on_message=on_message, + ) + + # -- Executor streaming (via Hummingbot WS) -- + + async def _executor_stream(self, channel: str) -> None: + parts = channel.split(":") + if len(parts) < 2: + return + server_name = parts[1] + + from config_manager import get_config_manager + + cm = get_config_manager() + + # Try SDS cache first (pre-warmed by auto_subscribe_servers or REST prefetch) + from condor.server_data_service import ServerDataType, get_server_data_service + + if channel not in self._last_data: + sds = get_server_data_service() + cached = sds.get(server_name, ServerDataType.EXECUTORS) + if cached is not None: + executors = self._transform_executors(cached) + if executors: + await self.broadcast(channel, executors) + logger.info( + "Executor SDS cache hit: %d executors for %s", + len(executors), + channel, + ) + + # Wait briefly for SDS to be populated by a concurrent REST request + # (usePrefetchData fires getExecutors which calls get_or_fetch on SDS) + if channel not in self._last_data: + sds = get_server_data_service() + for _ in range(6): # up to 3 seconds + await asyncio.sleep(0.5) + cached = sds.get(server_name, ServerDataType.EXECUTORS) + if cached is not None: + executors = self._transform_executors(cached) + if executors: + await self.broadcast(channel, executors) + logger.info( + "Executor SDS cache populated during wait: %d executors for %s", + len(executors), + channel, + ) + break + + # Progressive pre-fetch only if we still have no data + if channel not in self._last_data: + try: + from condor.fetchers._pagination import walk_pages + from condor.fetchers.executors import ( + extract_executors_list as _extract_executors_list, + ) + + sds = get_server_data_service() + client = await cm.get_client(server_name) + all_raw: list[dict] = [] + # Rows already through ExecutorInfo, accumulated page by page. + # Transforming the *whole* accumulated list on every page made + # this quadratic: with FIRST_PAGE/NEXT_PAGE/MAX_PREFETCH below, + # a 5k history cost ~27k pydantic validations instead of 5k, all + # on the event loop the candle and bots broadcasts share. + # ``_transform_executors`` is a pure per-item map over a list + # (``extract_executors_list`` passes a list through untouched), + # so transforming each page once yields the identical snapshot. + transformed: list[dict] = [] + page_num = 0 + FIRST_PAGE = 50 + NEXT_PAGE = 500 + MAX_PREFETCH = 5000 + + # A small first page so the tab paints before the long pages land. + async for page in walk_pages( + client.executors.search_executors, + _extract_executors_list, + page_size=NEXT_PAGE, + first_page_size=FIRST_PAGE, + max_items=MAX_PREFETCH, + ): + all_raw.extend(page) + transformed.extend(self._transform_executors(page)) + + # Broadcast the accumulated snapshot after each page. It is + # copied because ``broadcast`` retains what it is handed as + # the channel's last-known payload, which the next page + # would otherwise keep mutating underneath it. + if transformed: + await self.broadcast(channel, list(transformed)) + logger.info( + "Executor pre-fetch page %d: %d executors (total %d) for %s", + page_num, + len(page), + len(transformed), + channel, + ) + page_num += 1 + + # Cache in SDS so other consumers benefit + sds.put(server_name, ServerDataType.EXECUTORS, all_raw) + except asyncio.CancelledError: + return + except Exception as e: + logger.warning("Executor pre-fetch failed for %s: %s", channel, e) + + async def subscribe(ws: Any) -> None: + await ws.subscribe_executors(update_interval=2.0) + + async def on_message(msg: dict) -> None: + if msg.get("type") != "executors": + return + executors = self._transform_executors(msg.get("data", [])) + await self._broadcast_update(channel, executors) + + await self._run_ws_stream( + channel, + server_name, + label="Executor", + open_ws=lambda client: client.ws.executors(), + subscribe=subscribe, + on_message=on_message, + backoff_on_empty_close=True, + ) + + # -- Bots WS streaming (via Hummingbot /ws/executors all_bots_status) -- + + async def _bots_ws_stream(self, channel: str) -> None: + """Stream all_bots_status from Hummingbot /ws/executors and update SDS cache.""" + parts = channel.split(":") + if len(parts) < 2: + return + server_name = parts[1] + + # Send SDS-cached bots data as initial snapshot + if channel not in self._last_data: + from condor.server_data_service import ( + ServerDataType, + get_server_data_service, + ) + + sds = get_server_data_service() + cached = sds.get(server_name, ServerDataType.BOTS_STATUS) + if cached is not None: + try: + data = self._transform_bots(cached) + await self.broadcast(channel, data) + except Exception as e: + logger.debug("Failed to send initial bots snapshot: %s", e) + + async def subscribe(ws: Any) -> None: + # all_bots_status is not in the client library, send raw + await ws._send( + { + "action": "subscribe", + "type": "all_bots_status", + "update_interval": 5.0, + } + ) + resp = await ws._receive() + if resp.get("type") == "error": + raise RuntimeError(f"Subscribe failed: {resp.get('message')}") + + async def on_message(msg: dict) -> None: + if msg.get("type") != "all_bots_status": + return + raw_data = msg.get("data", {}) + # Update SDS cache so REST and Telegram benefit + from condor.server_data_service import ( + ServerDataType, + get_server_data_service, + ) + + get_server_data_service().put( + server_name, ServerDataType.BOTS_STATUS, raw_data + ) + try: + data = self._transform_bots(raw_data) + self._overlay_stopping_state(server_name, data) + await self._broadcast_update(channel, data) + except Exception as e: + logger.debug("Failed to transform bots WS data: %s", e) + + await self._run_ws_stream( + channel, + server_name, + label="Bots WS", + open_ws=lambda client: client.ws.executors(), + subscribe=subscribe, + on_message=on_message, + ) + + # -- Positions WS streaming (via Hummingbot /ws/executors positions) -- + + async def _positions_ws_stream(self, channel: str) -> None: + """Stream positions from Hummingbot /ws/executors and update SDS cache.""" + parts = channel.split(":") + if len(parts) < 2: + return + server_name = parts[1] + + async def subscribe(ws: Any) -> None: + await ws.subscribe_positions(update_interval=5.0) + + async def on_message(msg: dict) -> None: + if msg.get("type") != "positions": + return + raw_data = msg.get("data", []) + # Update SDS cache + from condor.server_data_service import ( + ServerDataType, + get_server_data_service, + ) + + get_server_data_service().put( + server_name, ServerDataType.POSITIONS, raw_data + ) + await self._broadcast_update(channel, raw_data) + + await self._run_ws_stream( + channel, + server_name, + label="Positions WS", + open_ws=lambda client: client.ws.executors(), + subscribe=subscribe, + on_message=on_message, + ) + + # -- Performance WS streaming (via Hummingbot /ws/executors performance) -- + + async def _performance_ws_stream(self, channel: str) -> None: + """Stream performance from Hummingbot /ws/executors and update SDS cache.""" + parts = channel.split(":") + if len(parts) < 2: + return + server_name = parts[1] + + async def subscribe(ws: Any) -> None: + await ws.subscribe_performance(update_interval=5.0) + + async def on_message(msg: dict) -> None: + if msg.get("type") != "performance": + return + await self._broadcast_update(channel, msg.get("data", {})) + + await self._run_ws_stream( + channel, + server_name, + label="Performance WS", + open_ws=lambda client: client.ws.executors(), + subscribe=subscribe, + on_message=on_message, + ) + + # -- Controller Performance polling stream -- + + async def _controller_perf_stream(self, channel: str) -> None: + """Poll latest controller performance every 30s and broadcast snapshots.""" + parts = channel.split(":") + if len(parts) < 2: + return + server_name = parts[1] + + from config_manager import get_config_manager + + cm = get_config_manager() + backoff = 5 + + while True: + try: + if not self._has_subscribers(channel): + logger.info( + "No subscribers for %s, stopping controller perf stream", + channel, + ) + self._controller_perf_tasks.pop(channel, None) + return + + client = await cm.get_client(server_name) + result = ( + await client.bot_orchestration.get_latest_controller_performance() + ) + + # Normalize to list of snapshots + snapshots = [] + if isinstance(result, list): + snapshots = result + elif isinstance(result, dict): + data = result.get( + "data", result.get("snapshots", result.get("records", [])) + ) + if isinstance(data, list): + snapshots = data + elif isinstance(data, dict): + for key, val in data.items(): + if isinstance(val, dict): + val.setdefault("controller_id", key) + snapshots.append(val) + + if snapshots: + await self._broadcast_update(channel, {"snapshots": snapshots}) + backoff = 5 + + await asyncio.sleep(30) + + except asyncio.CancelledError: + return + except Exception as e: + logger.warning( + "Controller perf stream error for %s: %s, retrying in %ds...", + channel, + e, + backoff, + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 120) diff --git a/condor/web/ws_manager.py b/condor/web/ws_manager.py index 6122c5f37..5e886b317 100644 --- a/condor/web/ws_manager.py +++ b/condor/web/ws_manager.py @@ -1,18 +1,35 @@ +"""WebSocket connection registry and channel broadcaster for the dashboard. + +The upstream stream implementations live in ``condor.web.streams`` (candles in +``streams.candles``, the Hummingbot-bridged protocols in +``streams.hummingbot_ws``) as mixins over the small host surface this module +implements — see ``condor.web.streams.StreamHost``. ``WebSocketManager`` keeps +only connection/subscription bookkeeping, the SDS bridge, broadcasting and the +stream lifecycle registry. +""" + from __future__ import annotations import asyncio import json import logging -import math import time -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional +from typing import TYPE_CHECKING, Any, Optional from fastapi import WebSocket -from condor import dex_candles +# Re-exported for back-compat: tests and callers import the candle helpers +# (and patch ``ws_manager.dex_candles``) through this module. +from condor import dex_candles # noqa: F401 from condor.asyncutil import TaskSet -from condor.fetchers.market_data import fetch_historical_candles, normalize_candle from condor.web.auth import decode_jwt +from condor.web.streams.candles import ( # noqa: F401 + CandleStreamsMixin, + _CandleBuffer, + _coerce_duration, + parse_candle_channel, +) +from condor.web.streams.hummingbot_ws import HummingbotStreamsMixin if TYPE_CHECKING: from condor.server_data_service import CacheKey @@ -73,135 +90,6 @@ def channel_for_key(key: CacheKey) -> str | None: return ":".join([prefix, key.server, *segments]) -# Interval string -> seconds for buffer sizing -_INTERVAL_SECONDS: dict[str, int] = { - "1s": 1, - "5s": 5, - "15s": 15, - "30s": 30, - "1m": 60, - "3m": 180, - "5m": 300, - "15m": 900, - "30m": 1800, - "1h": 3600, - "2h": 7200, - "4h": 14400, - "1d": 86400, - "1w": 604800, -} - -# Auto-cleanup candle buffers unused for this long -_CANDLE_BUFFER_IDLE_TTL = 600 # 10 minutes - - -def _coerce_duration(value: object) -> int | None: - """Coerce a client-supplied candle duration to a positive int (seconds). - - WS messages are untrusted input: a non-numeric value must not raise, or the - ValueError would escape handle_message and tear down the client's entire - multiplexed connection. Returns None for missing/invalid/non-positive values. - """ - if value is None or isinstance(value, bool): - return None - try: - duration = int(float(value)) - except (TypeError, ValueError): - logger.warning("Ignoring invalid candle duration from client: %r", value) - return None - return duration if duration > 0 else None - - -def parse_candle_channel(channel: str) -> tuple[str, str, str, str, str | None] | None: - """Split a candle channel into its parts, or ``None`` if it is malformed. - - The channel is ``candles:{server}:{connector}:{pair}:{interval}[:{pool_address}]``. - The pool address is an **optional sixth segment**, set by the DEX page so a - chart is pinned to the exact pool the user opened instead of whichever pool - ``dex_candles._resolve_pool`` judges deepest for the pair. A five-part channel - (every CLOB chart) parses to ``pool_address=None`` and is byte-identical to - what it was before the segment existed. - - Every consumer must go through here: unpacking five names off ``split(":")`` - raises ``ValueError`` on a six-part channel, and each of the three call sites - fails silently in a different way (a stream that never backfills, a backfill - that never ticks). - """ - parts = channel.split(":") - if len(parts) < 5: - return None - server_name, connector, pair, interval = parts[1:5] - pool_address = parts[5] if len(parts) > 5 else None - return server_name, connector, pair, interval, pool_address or None - - -class _CandleBuffer: - """Per-channel candle buffer with dynamic sizing based on interval + duration.""" - - __slots__ = ("interval", "_data", "_max_size", "last_accessed") - - def __init__(self, interval: str, duration_seconds: int = 3600): - self.interval = interval - self._data: dict[float, dict] = {} - self._max_size: int = 200 - self.last_accessed: float = time.monotonic() - self.set_duration(duration_seconds) - - def set_duration(self, duration_seconds: int) -> int: - """Resize buffer for the given duration. Returns the new max size.""" - interval_sec = _INTERVAL_SECONDS.get(self.interval, 60) - needed = math.ceil(duration_seconds / interval_sec) - new_max = max(needed, 200) # minimum 200 - old_max = self._max_size - self._max_size = new_max - self.last_accessed = time.monotonic() - self._evict() - if new_max != old_max: - logger.debug( - "Candle buffer resized %s: %d -> %d (duration=%ds)", - self.interval, - old_max, - new_max, - duration_seconds, - ) - return new_max - - def upsert(self, candle: dict) -> None: - self._data[candle["timestamp"]] = candle - self._evict() - - def upsert_many(self, candles: list[dict]) -> None: - for c in candles: - self._data[c["timestamp"]] = c - self._evict() - - def get_sorted(self) -> list[dict]: - self.last_accessed = time.monotonic() - return sorted(self._data.values(), key=lambda c: c["timestamp"]) - - def _evict(self) -> None: - excess = len(self._data) - self._max_size - if excess <= 0: - return - # Drop the `excess` oldest timestamps in a single O(n log n) pass - # instead of calling min() (O(n)) per evicted candle. - for ts in sorted(self._data)[:excess]: - del self._data[ts] - - @property - def size(self) -> int: - return len(self._data) - - @property - def max_size(self) -> int: - return self._max_size - - @property - def needs_backfill(self) -> bool: - """True if buffer has room for significantly more candles.""" - return len(self._data) < self._max_size * 0.5 - - class _Connection: __slots__ = ("ws", "user_id", "channels") @@ -211,16 +99,17 @@ def __init__(self, ws: WebSocket, user_id: int): self.channels: set[str] = set() -class WebSocketManager: +class WebSocketManager(CandleStreamsMixin, HummingbotStreamsMixin): """Manages WebSocket connections and channel-based data broadcasting. Subscribes to ServerDataService for data updates and broadcasts - to connected WebSocket clients. Candle streaming uses dedicated - WebSocket connections with dynamic per-channel buffering. + to connected WebSocket clients. The upstream stream protocols are + supplied by the mixins (``CandleStreamsMixin``, + ``HummingbotStreamsMixin``); this class owns their shared state and + the host surface they run on (``broadcast``, ``_send``, + ``_has_subscribers``, ``_ensure_stream``, ``_oneshot_tasks``). """ - _CANDLE_KEEP_ALIVE = 300 # 5-minute grace period before tearing down candle streams - def __init__(self): self._connections: list[_Connection] = [] self._last_data: dict[str, Any] = {} # channel -> last broadcast payload @@ -264,11 +153,6 @@ def _server_from_channel(channel: str) -> str | None: parts = channel.split(":") return parts[1] if len(parts) >= 2 and parts[1] else None - @staticmethod - def _normalize_candle(c: Any) -> dict | None: - """Normalize a candle from any format to a uniform dict with float values.""" - return normalize_candle(c) - # -- Lifecycle -- def start(self) -> None: @@ -450,135 +334,7 @@ async def handle_message(self, conn: _Connection, raw: str) -> None: if channel.startswith("candles:") and duration: await self._handle_candle_duration_change(conn, channel, duration) - async def _handle_candle_subscribe( - self, conn: _Connection, channel: str, duration: object - ) -> None: - """Handle candle channel subscription with optional duration.""" - parts = channel.split(":") - if len(parts) < 5: - return - interval = parts[4] - dur = _coerce_duration(duration) or 3 * 86400 # default 3 days - - # Cancel any pending teardown — a subscriber just came back - timer = self._candle_teardown_timers.pop(channel, None) - if timer is not None: - timer.cancel() - logger.debug("Cancelled pending candle teardown for %s", channel) - - buf = self._candle_buffers.get(channel) - if buf is None: - buf = _CandleBuffer(interval, dur) - self._candle_buffers[channel] = buf - else: - # Expand buffer if this client needs more - if dur > 0: - old_max = buf.max_size - buf.set_duration(dur) - if buf.max_size > old_max and buf.needs_backfill: - self._oneshot_tasks.track( - asyncio.create_task( - self._backfill_candles(channel), - name=f"backfill:{channel}", - ) - ) - - # Send buffered candles as initial snapshot - sorted_candles = buf.get_sorted() - if sorted_candles: - await self._send(conn, channel, {"type": "candles", "data": sorted_candles}) - - # If the stream task is still running (kept alive during grace period), skip restart - self._ensure_stream("candles", channel) - - async def _handle_candle_duration_change( - self, conn: _Connection, channel: str, duration: int - ) -> None: - """Handle duration change for an existing candle subscription. - - Only grows the buffer — never shrinks. The frontend manages its own - display window; the backend just ensures enough history is buffered. - """ - buf = self._candle_buffers.get(channel) - if buf is None: - return - old_max = buf.max_size - # Only expand — skip if requested duration would shrink the buffer - interval_sec = _INTERVAL_SECONDS.get(buf.interval, 60) - needed = max(math.ceil(duration / interval_sec), 200) - if needed <= old_max: - return - buf.set_duration(duration) - if buf.needs_backfill: - await self._backfill_candles(channel) - # Broadcast updated snapshot to ALL subscribers on this channel - sorted_candles = buf.get_sorted() - if sorted_candles: - await self.broadcast(channel, {"type": "candles", "data": sorted_candles}) - - async def _backfill_candles(self, channel: str) -> None: - """Fetch historical candles to fill the buffer gap.""" - parsed = parse_candle_channel(channel) - if parsed is None: - return - server_name, connector, pair, interval, pool_address = parsed - buf = self._candle_buffers.get(channel) - if buf is None: - return - - from config_manager import get_config_manager - - cm = get_config_manager() - try: - interval_sec = _INTERVAL_SECONDS.get(interval, 60) - end_time = int(time.time()) - start_time = end_time - (buf.max_size * interval_sec) - - logger.info( - "Backfilling candles for %s: need %d, have %d, fetching %ds-%ds", - channel, - buf.max_size, - buf.size, - start_time, - end_time, - ) - - if dex_candles.uses_gecko_candles(connector): - candles = await dex_candles.fetch_dex_candles( - connector, pool_address, pair, interval, start_time, end_time - ) - if candles: - buf.upsert_many(candles) - logger.info( - "Backfilled %d GeckoTerminal candles for %s (buffer: %d/%d)", - len(candles), - channel, - buf.size, - buf.max_size, - ) - return - - client = await cm.get_client(server_name) - candles = await fetch_historical_candles( - client, - connector, - pair, - interval, - start_time=start_time, - end_time=end_time, - limit=min(buf.max_size, 5000), - ) - if candles: - buf.upsert_many(candles) - logger.info( - "Backfilled %d candles for %s (buffer: %d/%d)", - len(candles), - channel, - buf.size, - buf.max_size, - ) - except Exception as e: - logger.warning("Candle backfill failed for %s: %s", channel, e) + # -- SDS bridge -- async def _subscribe_sds(self, channel: str) -> None: """Subscribe to SDS for a channel and prime cache.""" @@ -699,36 +455,6 @@ def _unsubscribe_portfolio_history(self, server_name: str) -> None: # -- SDS listener -- - @staticmethod - def _transform_executors(raw_data: Any) -> list[dict]: - """Transform raw executor data to ExecutorInfo-compatible dicts for WS broadcast.""" - from condor.fetchers.executors import extract_executors_list - from condor.web.models import ExecutorInfo - - executors_list = extract_executors_list(raw_data) - result = [] - for ex in executors_list: - info = ExecutorInfo.from_raw(ex) - if info: - result.append(info.model_dump()) - return result - - @staticmethod - def _transform_bots(raw_data: Any) -> dict: - """Transform raw BOTS_STATUS data to BotsPageResponse-compatible dict for WS broadcast.""" - from condor.fetchers.bots import build_bots_page - - return build_bots_page(raw_data) - - @staticmethod - def _overlay_stopping_state(server_name: str, data: dict) -> None: - """Apply transitional 'stopping' state to WS broadcast data.""" - from condor.web.routes.bots import overlay_stopping_state - - overlay_stopping_state( - server_name, data.get("controllers", []), data.get("bots", []) - ) - def _on_data_update(self, key: CacheKey, value: Any) -> None: """Called by SDS on every cache write. Maps the key to a WS channel and broadcasts. ``key`` is an SDS ``CacheKey``: server, data type and params.""" @@ -891,913 +617,6 @@ def _maybe_stop_stream(self, prefix: str, channel: str) -> None: logger.info(spec["stop_log"], channel) self._last_data.pop(channel, None) - def _maybe_stop_candle_stream(self, channel: str) -> None: - # If subscribers still exist, cancel any pending teardown and return - if self._has_subscribers(channel): - timer = self._candle_teardown_timers.pop(channel, None) - if timer is not None: - timer.cancel() - return - - # Already have a pending teardown scheduled — nothing to do - if channel in self._candle_teardown_timers: - return - - # Schedule deferred teardown - loop = asyncio.get_event_loop() - handle = loop.call_later( - self._CANDLE_KEEP_ALIVE, - self._deferred_stop_candle_stream, - channel, - ) - self._candle_teardown_timers[channel] = handle - logger.info( - "Scheduled candle stream teardown for %s in %ds", - channel, - self._CANDLE_KEEP_ALIVE, - ) - - def _deferred_stop_candle_stream(self, channel: str) -> None: - """Actually tear down a candle stream after the grace period.""" - self._candle_teardown_timers.pop(channel, None) - - # Re-check: subscribers may have appeared during the grace period - if self._has_subscribers(channel): - logger.debug( - "Candle teardown cancelled — subscribers returned for %s", channel - ) - return - - task = self._candle_tasks.pop(channel, None) - if task and not task.done(): - task.cancel() - logger.info("Deferred candle stream teardown completed for %s", channel) - poll_task = self._candle_poll_tasks.pop(channel, None) - if poll_task and not poll_task.done(): - poll_task.cancel() - self._last_candle_ws_update.pop(channel, None) - self._candle_first_msg_logged.discard(channel) - # Snapshot-on-resubscribe is served by _candle_buffers, not _last_data - self._last_data.pop(channel, None) - # NOTE: candle buffer is NOT deleted — the existing idle cleanup loop handles that - - @staticmethod - def _is_permanent_ws_error(error_str: str) -> bool: - """True if a stream error will never succeed on retry. - - Besides auth/not-found (401/403/404), an invalid trading pair (wrong - symbol for this connector) will never become valid, so retrying - forever just spams logs and hammers the exchange until Condor - restarts (issue #134). - """ - lowered = error_str.lower() - return ( - any(code in error_str for code in ("401", "403", "404")) - or "appears to be invalid" in lowered - or "invalid symbol" in lowered - ) - - async def _run_ws_stream( - self, - channel: str, - server_name: str, - *, - label: str, - open_ws: Callable[[Any], Any], - subscribe: Callable[[Any], Awaitable[None]], - on_message: Callable[[dict], Awaitable[None]], - backoff_on_empty_close: bool = False, - ) -> None: - """Shared connect/reconnect skeleton for all Hummingbot WS streams. - - Owns the while-True loop, subscriber check, heartbeat/error message - handling, permanent-error detection (``_is_permanent_ws_error``) and - exponential backoff capped at 60s. Each stream supplies only: - - - ``open_ws``: client -> the WS async context manager to enter - (e.g. ``lambda c: c.ws.market_data()``). - - ``subscribe``: performs the subscription on the open socket. - - ``on_message``: handles data messages (heartbeat/error are - handled here). - - ``backoff_on_empty_close`` reproduces the executor stream's - behavior: don't reset backoff on subscribe; after a clean close, - reset it only if at least one message arrived, otherwise back off - (guards against a server that accepts then immediately drops). - """ - from config_manager import get_config_manager - - cm = get_config_manager() - backoff = 5 - lower_label = label[0].lower() + label[1:] - subscribed_label = label if label.endswith("WS") else f"{label} WS" - - while True: - try: - client = await cm.get_client(server_name) - async with open_ws(client) as ws: - await subscribe(ws) - logger.info("%s subscribed: %s", subscribed_label, channel) - if not backoff_on_empty_close: - backoff = 5 - got_message = False - async for msg in ws: - if not self._has_subscribers(channel): - logger.info( - "No subscribers for %s, closing %s stream", - channel, - lower_label, - ) - return - - got_message = True - msg_type = msg.get("type") - if msg_type == "heartbeat": - continue - if msg_type == "error": - logger.warning( - "%s stream error for %s: %s", - label, - channel, - msg.get("message", "unknown error"), - ) - break - await on_message(msg) - - if backoff_on_empty_close: - # Connection closed cleanly — back off if short-lived - if got_message: - backoff = 5 - else: - logger.warning( - "%s stream closed immediately for %s, " - "reconnecting in %ds...", - label, - channel, - backoff, - ) - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 60) - - except asyncio.CancelledError: - return - except Exception as e: - if self._is_permanent_ws_error(str(e)): - logger.warning( - "%s stream permanent error for %s: %s — giving up", - label, - channel, - e, - ) - return - - logger.warning( - "%s stream error for %s: %s, reconnecting in %ds...", - label, - channel, - e, - backoff, - ) - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 60) - - async def _candle_stream(self, channel: str) -> None: - parsed = parse_candle_channel(channel) - if parsed is None: - return - server_name, connector, pair, interval, pool_address = parsed - - from config_manager import get_config_manager - - cm = get_config_manager() - backoff = 5 - - # Start REST poll fallback alongside the WS stream - self._ensure_candle_poll_fallback(channel) - - if dex_candles.uses_gecko_candles(connector): - # No CandlesFactory feed to subscribe to (xrpl, DEX networks): the - # poll loop above is the whole stream. Subscribing anyway would just - # reconnect-loop on an error the API can never stop returning. - logger.info( - "Candle stream for %s is GeckoTerminal-polled (no API feed)", channel - ) - return - - while True: - try: - client = await cm.get_client(server_name) - async with client.ws.market_data() as ws: - await ws.subscribe_candles( - connector, - pair, - interval=interval, - max_records=100, - update_interval=1.0, - ) - logger.info("Candle WS subscribed: %s", channel) - backoff = 5 # Reset on successful connection - async for msg in ws: - # Log first message per channel at INFO for diagnostics - if channel not in self._candle_first_msg_logged: - self._candle_first_msg_logged.add(channel) - logger.info( - "Candle WS first message for %s: %s", - channel, - ( - json.dumps(msg)[:500] - if isinstance(msg, dict) - else type(msg).__name__ - ), - ) - else: - logger.debug( - "Candle WS raw msg keys for %s: %s", - channel, - ( - list(msg.keys()) - if isinstance(msg, dict) - else type(msg).__name__ - ), - ) - - msg_type = msg.get("type") - if msg_type == "candle_update": - raw = msg.get("data") - candle = self._normalize_candle(raw) if raw else None - if candle: - self._last_candle_ws_update[channel] = time.monotonic() - self._upsert_candle_buffer(channel, candle) - await self.broadcast( - channel, - {"type": "candle_update", "candle": candle}, - ) - elif msg_type == "candles": - raw_list = msg.get("data") or [] - candles = [ - c - for r in raw_list - if (c := self._normalize_candle(r)) is not None - ] - if candles: - self._last_candle_ws_update[channel] = time.monotonic() - self._upsert_candle_buffer_many(channel, candles) - await self.broadcast( - channel, - {"type": "candles", "data": candles}, - ) - elif msg_type in ("heartbeat", "subscribed"): - continue - elif msg_type == "error": - error_msg = msg.get("message", "unknown error") - logger.warning( - "Candle stream error for %s: %s — continuing", - channel, - error_msg, - ) - await self.broadcast( - channel, - { - "type": "error", - "message": f"Stream error: {error_msg}", - }, - ) - # Don't break — the WS may still be alive. - # If truly dead, next recv raises and we reconnect. - continue - else: - logger.info( - "Candle stream unrecognized msg type for %s: type=%s keys=%s", - channel, - msg_type, - ( - list(msg.keys()) - if isinstance(msg, dict) - else type(msg).__name__ - ), - ) - - except asyncio.CancelledError: - return - except Exception as e: - error_str = str(e) - # Detect permanent failures — don't retry (issue #134, see - # _is_permanent_ws_error). - if self._is_permanent_ws_error(error_str): - logger.warning( - "Candle stream permanent error for %s: %s — giving up", - channel, - e, - ) - await self.broadcast( - channel, - {"type": "error", "message": f"Stream failed: {error_str}"}, - ) - # Stop the paired REST poll fallback — nothing valid to poll. - poll_task = self._candle_poll_tasks.pop(channel, None) - if poll_task and not poll_task.done(): - poll_task.cancel() - return - - logger.warning( - "Candle stream error for %s: %s, reconnecting in %ds...", - channel, - e, - backoff, - ) - await self.broadcast( - channel, - { - "type": "error", - "message": f"Connection lost, retrying in {backoff}s", - }, - ) - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 60) - - # -- Candle REST poll fallback -- - - def _ensure_candle_poll_fallback(self, channel: str) -> None: - """Start a REST poll fallback task for a candle channel if not already running.""" - if ( - channel in self._candle_poll_tasks - and not self._candle_poll_tasks[channel].done() - ): - return - self._candle_poll_tasks[channel] = asyncio.create_task( - self._candle_poll_fallback(channel) - ) - - async def _candle_poll_fallback(self, channel: str) -> None: - """Periodically poll REST for candles when WS stream goes silent. - - For a GeckoTerminal-backed connector there is no WS stream to fall back - from — this loop *is* the feed, so it polls from the first tick and never - waits for staleness. - """ - parsed = parse_candle_channel(channel) - if parsed is None: - return - server_name, connector, pair, interval, pool_address = parsed - interval_sec = _INTERVAL_SECONDS.get(interval, 60) - gecko = dex_candles.uses_gecko_candles(connector) - # How long without a WS update before we consider it stale - stale_threshold = max(interval_sec, 15) - # How often to poll REST once stale (keep the last candle fresh). Gecko - # polls stay well clear of its per-minute rate limit — every chart on the - # dashboard shares one budget — while still refreshing a forming candle. - poll_interval = ( - max(min(interval_sec // 2, 60), 30) if gecko else min(interval_sec, 10) - ) - was_stale = gecko - - from config_manager import get_config_manager - - try: - while True: - await asyncio.sleep(poll_interval if was_stale else stale_threshold) - - last_update = self._last_candle_ws_update.get(channel) - if ( - last_update is not None - and (time.monotonic() - last_update) <= stale_threshold - ): - if was_stale: - logger.info( - "Candle WS stream resumed for %s, stopping REST fallback polling", - channel, - ) - was_stale = False - continue - - # Stream is stale — poll REST - if not was_stale: - logger.warning( - "Candle WS stream appears stale for %s (threshold=%ds), polling REST fallback", - channel, - stale_threshold, - ) - was_stale = True - - try: - now = int(time.time()) - if gecko: - candles = await dex_candles.fetch_dex_candles( - connector, - pool_address, - pair, - interval, - now - interval_sec * 5, - now, - # The shared OHLCV cache holds a pool for minutes, - # long enough to freeze a live chart. - use_cache=False, - ) - else: - cm = get_config_manager() - client = await cm.get_client(server_name) - # No `limit`: this poll only wants the fresh tail of the - # live range, so an empty historical answer stays empty - # rather than pulling a full unrelated window. - candles = await fetch_historical_candles( - client, - connector, - pair, - interval, - start_time=now - interval_sec * 5, - end_time=now, - ) - if candles: - buf = self._candle_buffers.get(channel) - # Broadcast if we have newer candles OR if the latest - # candle's OHLCV changed (same timestamp, updated values) - newest_buf_ts = ( - max(buf._data.keys()) if buf and buf._data else 0 - ) - newest_poll_ts = max(c["timestamp"] for c in candles) - changed = newest_poll_ts > newest_buf_ts - if not changed and buf and newest_poll_ts in buf._data: - # Same timestamp — check if OHLCV actually changed - old = buf._data[newest_poll_ts] - new = next( - c for c in candles if c["timestamp"] == newest_poll_ts - ) - changed = any( - old.get(k) != new.get(k) - for k in ("open", "high", "low", "close", "volume") - ) - if changed: - self._upsert_candle_buffer_many(channel, candles) - await self.broadcast( - channel, - {"type": "candles", "data": candles}, - ) - logger.debug( - "REST fallback delivered %d candles for %s", - len(candles), - channel, - ) - except Exception as e: - logger.debug("REST candle poll failed for %s: %s", channel, e) - - except asyncio.CancelledError: - return - - # -- Trade streaming -- - - async def _trade_stream(self, channel: str) -> None: - parts = channel.split(":") - if len(parts) < 4: - return - _, server_name, connector, pair = parts[:4] - - async def subscribe(ws: Any) -> None: - await ws.subscribe_trades( - connector, - pair, - update_interval=1.0, - ) - - async def on_message(msg: dict) -> None: - if msg.get("type") != "trades": - return - trade_data = msg.get("data", []) - trades = [] - for t in trade_data: - if isinstance(t, dict): - trades.append( - { - "price": float(t.get("price", 0)), - "amount": float(t.get("amount", t.get("quantity", 0))), - "side": t.get("side", t.get("trade_type", "buy")).lower(), - "timestamp": float(t.get("timestamp", 0)), - } - ) - if trades: - await self.broadcast(channel, {"type": "trades", "data": trades}) - - await self._run_ws_stream( - channel, - server_name, - label="Trade", - open_ws=lambda client: client.ws.market_data(), - subscribe=subscribe, - on_message=on_message, - ) - - # -- Candle buffer helpers -- - - def _get_or_create_candle_buffer(self, channel: str) -> _CandleBuffer: - """Get existing buffer or create with default duration.""" - buf = self._candle_buffers.get(channel) - if buf is None: - parts = channel.split(":") - interval = parts[4] if len(parts) >= 5 else "1m" - buf = _CandleBuffer(interval, 3 * 86400) # default 3 days - self._candle_buffers[channel] = buf - return buf - - def _upsert_candle_buffer(self, channel: str, candle: dict) -> None: - """Upsert a single candle into the per-channel buffer.""" - buf = self._get_or_create_candle_buffer(channel) - buf.upsert(candle) - - def _upsert_candle_buffer_many(self, channel: str, candles: list[dict]) -> None: - """Upsert multiple candles into the per-channel buffer.""" - buf = self._get_or_create_candle_buffer(channel) - buf.upsert_many(candles) - - async def _candle_buffer_cleanup_loop(self) -> None: - """Periodically remove candle buffers that haven't been accessed.""" - try: - while True: - await asyncio.sleep(60) # check every minute - now = time.monotonic() - stale = [ - ch - for ch, buf in self._candle_buffers.items() - if (now - buf.last_accessed) > _CANDLE_BUFFER_IDLE_TTL - and not any(ch in c.channels for c in self._connections) - ] - for ch in stale: - buf = self._candle_buffers.pop(ch, None) - if buf: - logger.info( - "Cleaned up idle candle buffer: %s (%d candles)", - ch, - buf.size, - ) - except asyncio.CancelledError: - return - - # -- Order book streaming -- - - async def _order_book_stream(self, channel: str) -> None: - parts = channel.split(":") - if len(parts) < 4: - return - _, server_name, connector, pair = parts[:4] - - async def subscribe(ws: Any) -> None: - await ws.subscribe_order_book( - connector, - pair, - depth=20, - update_interval=1.0, - ) - - async def on_message(msg: dict) -> None: - if msg.get("type") != "order_book": - return - raw_data = msg.get("data", {}) - bids = [] - asks = [] - for b in raw_data.get("bids") or []: - if isinstance(b, dict): - bids.append( - { - "price": float(b.get("price", 0)), - "amount": float(b.get("amount", b.get("quantity", 0))), - } - ) - elif isinstance(b, (list, tuple)) and len(b) >= 2: - bids.append({"price": float(b[0]), "amount": float(b[1])}) - for a in raw_data.get("asks") or []: - if isinstance(a, dict): - asks.append( - { - "price": float(a.get("price", 0)), - "amount": float(a.get("amount", a.get("quantity", 0))), - } - ) - elif isinstance(a, (list, tuple)) and len(a) >= 2: - asks.append({"price": float(a[0]), "amount": float(a[1])}) - await self.broadcast(channel, {"bids": bids, "asks": asks}) - - await self._run_ws_stream( - channel, - server_name, - label="Order book", - open_ws=lambda client: client.ws.market_data(), - subscribe=subscribe, - on_message=on_message, - ) - - # -- Executor streaming (via Hummingbot WS) -- - - async def _executor_stream(self, channel: str) -> None: - parts = channel.split(":") - if len(parts) < 2: - return - server_name = parts[1] - - from config_manager import get_config_manager - - cm = get_config_manager() - - # Try SDS cache first (pre-warmed by auto_subscribe_servers or REST prefetch) - from condor.server_data_service import ServerDataType, get_server_data_service - - if channel not in self._last_data: - sds = get_server_data_service() - cached = sds.get(server_name, ServerDataType.EXECUTORS) - if cached is not None: - executors = self._transform_executors(cached) - if executors: - await self.broadcast(channel, executors) - logger.info( - "Executor SDS cache hit: %d executors for %s", - len(executors), - channel, - ) - - # Wait briefly for SDS to be populated by a concurrent REST request - # (usePrefetchData fires getExecutors which calls get_or_fetch on SDS) - if channel not in self._last_data: - sds = get_server_data_service() - for _ in range(6): # up to 3 seconds - await asyncio.sleep(0.5) - cached = sds.get(server_name, ServerDataType.EXECUTORS) - if cached is not None: - executors = self._transform_executors(cached) - if executors: - await self.broadcast(channel, executors) - logger.info( - "Executor SDS cache populated during wait: %d executors for %s", - len(executors), - channel, - ) - break - - # Progressive pre-fetch only if we still have no data - if channel not in self._last_data: - try: - from condor.fetchers._pagination import walk_pages - from condor.fetchers.executors import ( - extract_executors_list as _extract_executors_list, - ) - - sds = get_server_data_service() - client = await cm.get_client(server_name) - all_raw: list[dict] = [] - # Rows already through ExecutorInfo, accumulated page by page. - # Transforming the *whole* accumulated list on every page made - # this quadratic: with FIRST_PAGE/NEXT_PAGE/MAX_PREFETCH below, - # a 5k history cost ~27k pydantic validations instead of 5k, all - # on the event loop the candle and bots broadcasts share. - # ``_transform_executors`` is a pure per-item map over a list - # (``extract_executors_list`` passes a list through untouched), - # so transforming each page once yields the identical snapshot. - transformed: list[dict] = [] - page_num = 0 - FIRST_PAGE = 50 - NEXT_PAGE = 500 - MAX_PREFETCH = 5000 - - # A small first page so the tab paints before the long pages land. - async for page in walk_pages( - client.executors.search_executors, - _extract_executors_list, - page_size=NEXT_PAGE, - first_page_size=FIRST_PAGE, - max_items=MAX_PREFETCH, - ): - all_raw.extend(page) - transformed.extend(self._transform_executors(page)) - - # Broadcast the accumulated snapshot after each page. It is - # copied because ``broadcast`` retains what it is handed as - # the channel's last-known payload, which the next page - # would otherwise keep mutating underneath it. - if transformed: - await self.broadcast(channel, list(transformed)) - logger.info( - "Executor pre-fetch page %d: %d executors (total %d) for %s", - page_num, - len(page), - len(transformed), - channel, - ) - page_num += 1 - - # Cache in SDS so other consumers benefit - sds.put(server_name, ServerDataType.EXECUTORS, all_raw) - except asyncio.CancelledError: - return - except Exception as e: - logger.warning("Executor pre-fetch failed for %s: %s", channel, e) - - async def subscribe(ws: Any) -> None: - await ws.subscribe_executors(update_interval=2.0) - - async def on_message(msg: dict) -> None: - if msg.get("type") != "executors": - return - executors = self._transform_executors(msg.get("data", [])) - await self._broadcast_update(channel, executors) - - await self._run_ws_stream( - channel, - server_name, - label="Executor", - open_ws=lambda client: client.ws.executors(), - subscribe=subscribe, - on_message=on_message, - backoff_on_empty_close=True, - ) - - # -- Bots WS streaming (via Hummingbot /ws/executors all_bots_status) -- - - async def _bots_ws_stream(self, channel: str) -> None: - """Stream all_bots_status from Hummingbot /ws/executors and update SDS cache.""" - parts = channel.split(":") - if len(parts) < 2: - return - server_name = parts[1] - - # Send SDS-cached bots data as initial snapshot - if channel not in self._last_data: - from condor.server_data_service import ( - ServerDataType, - get_server_data_service, - ) - - sds = get_server_data_service() - cached = sds.get(server_name, ServerDataType.BOTS_STATUS) - if cached is not None: - try: - data = self._transform_bots(cached) - await self.broadcast(channel, data) - except Exception as e: - logger.debug("Failed to send initial bots snapshot: %s", e) - - async def subscribe(ws: Any) -> None: - # all_bots_status is not in the client library, send raw - await ws._send( - { - "action": "subscribe", - "type": "all_bots_status", - "update_interval": 5.0, - } - ) - resp = await ws._receive() - if resp.get("type") == "error": - raise RuntimeError(f"Subscribe failed: {resp.get('message')}") - - async def on_message(msg: dict) -> None: - if msg.get("type") != "all_bots_status": - return - raw_data = msg.get("data", {}) - # Update SDS cache so REST and Telegram benefit - from condor.server_data_service import ( - ServerDataType, - get_server_data_service, - ) - - get_server_data_service().put( - server_name, ServerDataType.BOTS_STATUS, raw_data - ) - try: - data = self._transform_bots(raw_data) - self._overlay_stopping_state(server_name, data) - await self._broadcast_update(channel, data) - except Exception as e: - logger.debug("Failed to transform bots WS data: %s", e) - - await self._run_ws_stream( - channel, - server_name, - label="Bots WS", - open_ws=lambda client: client.ws.executors(), - subscribe=subscribe, - on_message=on_message, - ) - - # -- Positions WS streaming (via Hummingbot /ws/executors positions) -- - - async def _positions_ws_stream(self, channel: str) -> None: - """Stream positions from Hummingbot /ws/executors and update SDS cache.""" - parts = channel.split(":") - if len(parts) < 2: - return - server_name = parts[1] - - async def subscribe(ws: Any) -> None: - await ws.subscribe_positions(update_interval=5.0) - - async def on_message(msg: dict) -> None: - if msg.get("type") != "positions": - return - raw_data = msg.get("data", []) - # Update SDS cache - from condor.server_data_service import ( - ServerDataType, - get_server_data_service, - ) - - get_server_data_service().put( - server_name, ServerDataType.POSITIONS, raw_data - ) - await self._broadcast_update(channel, raw_data) - - await self._run_ws_stream( - channel, - server_name, - label="Positions WS", - open_ws=lambda client: client.ws.executors(), - subscribe=subscribe, - on_message=on_message, - ) - - # -- Performance WS streaming (via Hummingbot /ws/executors performance) -- - - async def _performance_ws_stream(self, channel: str) -> None: - """Stream performance from Hummingbot /ws/executors and update SDS cache.""" - parts = channel.split(":") - if len(parts) < 2: - return - server_name = parts[1] - - async def subscribe(ws: Any) -> None: - await ws.subscribe_performance(update_interval=5.0) - - async def on_message(msg: dict) -> None: - if msg.get("type") != "performance": - return - await self._broadcast_update(channel, msg.get("data", {})) - - await self._run_ws_stream( - channel, - server_name, - label="Performance WS", - open_ws=lambda client: client.ws.executors(), - subscribe=subscribe, - on_message=on_message, - ) - - # -- Controller Performance polling stream -- - - async def _controller_perf_stream(self, channel: str) -> None: - """Poll latest controller performance every 30s and broadcast snapshots.""" - parts = channel.split(":") - if len(parts) < 2: - return - server_name = parts[1] - - from config_manager import get_config_manager - - cm = get_config_manager() - backoff = 5 - - while True: - try: - if not self._has_subscribers(channel): - logger.info( - "No subscribers for %s, stopping controller perf stream", - channel, - ) - self._controller_perf_tasks.pop(channel, None) - return - - client = await cm.get_client(server_name) - result = ( - await client.bot_orchestration.get_latest_controller_performance() - ) - - # Normalize to list of snapshots - snapshots = [] - if isinstance(result, list): - snapshots = result - elif isinstance(result, dict): - data = result.get( - "data", result.get("snapshots", result.get("records", [])) - ) - if isinstance(data, list): - snapshots = data - elif isinstance(data, dict): - for key, val in data.items(): - if isinstance(val, dict): - val.setdefault("controller_id", key) - snapshots.append(val) - - if snapshots: - await self._broadcast_update(channel, {"snapshots": snapshots}) - backoff = 5 - - await asyncio.sleep(30) - - except asyncio.CancelledError: - return - except Exception as e: - logger.warning( - "Controller perf stream error for %s: %s, retrying in %ds...", - channel, - e, - backoff, - ) - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 120) - # -- Singleton -- diff --git a/config_manager.py b/config_manager.py index 38b0638c1..358d30924 100644 --- a/config_manager.py +++ b/config_manager.py @@ -1081,10 +1081,20 @@ def get_effective_server(chat_id: int, user_data: dict = None) -> str | None: """Get the effective default server for a chat, checking both user_data and config.yml. Priority: - 1. user_data active_server (from pickle, fast in-memory) - 2. chat_defaults from config.yml (persistent across hard kills) + 1. chat_defaults from config.yml — the one place *every* surface writes when + the user picks a default (Telegram's /servers, the web dashboard, admin + assignment), so it is the only entry that can be current. + 2. user_data active_server (from pickle, per-user, covers chats that never + had a default of their own) 3. None if nothing configured + The order used to be the other way round, and that made the web dashboard's + "set default" invisible to a live Telegram session: the web writes + config.yml, while the pickle in the running bot's ``user_data`` kept the + server chosen before it and won every lookup until the process restarted. + The pickle is a cache of the choice, not the choice — so it is read second + and re-synced from config.yml whenever the two disagree. + Args: chat_id: The chat ID user_data: Optional user_data dict from context @@ -1092,25 +1102,36 @@ def get_effective_server(chat_id: int, user_data: dict = None) -> str | None: Returns: Server name or None """ + from condor.preferences import SERVER_PIN_KEY from handlers.config.user_preferences import get_active_server - # First check user_data (pickle - might be lost on hard kill) - if user_data: - active = get_active_server(user_data) - if active: - return active + # A context built for one server — a routine or an agent run launched + # against it — carries its own answer and is never re-resolved from a chat's + # ambient default. Everything else reaching this point holds a preference, + # which is a cache of a choice config.yml records authoritatively. + if user_data and user_data.get(SERVER_PIN_KEY): + pinned = get_active_server(user_data) + if pinned: + return pinned - # Fall back to chat_defaults in config.yml (always persisted) cm = get_config_manager() chat_default = cm._data.get("chat_defaults", {}).get(chat_id) if chat_default and chat_default in cm._data.get("servers", {}): - # Also sync back to user_data for fast future access - if user_data is not None: + # Keep the pickle in step, but only on an actual change: the setter + # persists the whole preference section to config.yml, and this runs on + # every server lookup. + if user_data is not None and get_active_server(user_data) != chat_default: from handlers.config.user_preferences import set_active_server set_active_server(user_data, chat_default) return chat_default + # No default recorded for this chat — fall back to the user's own last pick. + if user_data: + active = get_active_server(user_data) + if active: + return active + return None diff --git a/frontend/src/components/TelemetryConsentBanner.tsx b/frontend/src/components/TelemetryConsentBanner.tsx new file mode 100644 index 000000000..acc45132c --- /dev/null +++ b/frontend/src/components/TelemetryConsentBanner.tsx @@ -0,0 +1,72 @@ +import { Loader2, ShieldCheck } from "lucide-react"; +import { Link } from "react-router-dom"; + +import { shouldAskConsent, useSetTelemetryLevel, useTelemetry } from "@/hooks/useTelemetry"; +import type { TelemetryLevel } from "@/lib/api"; + +/** + * The dashboard's half of the consent prompt. + * + * A Telegram install gets asked once, next to the boot notification, because + * that is the one moment the admin is already looking. A local-mode install has + * no bot to be asked through, so the equivalent moment is the first dashboard + * it opens — which is this strip. + * + * There is no dismiss, because both buttons are answers and one click retires + * it forever; a "later" would only add a way to be asked again. It renders for + * nobody else: not for a non-admin seat, not once answered on either surface, + * and not when `CONDOR_TELEMETRY` has already decided. + */ +export function TelemetryConsentBanner() { + const { data } = useTelemetry(); + const mutation = useSetTelemetryLevel(); + + if (!shouldAskConsent(data) || !data) return null; + + const { disclosure } = data; + + return ( +
+ + + {/* `min-w-0` so the copy wraps inside its own column instead of pushing + the buttons onto a row of their own. */} +

+ {disclosure.headline}{" "} + {disclosure.optional}{" "} + + What is never sent + +

+ +
+ {mutation.isPending && ( + + )} + {disclosure.options.map((option, index) => ( + + ))} +
+ + {mutation.isError && ( +

+ Could not save that. {(mutation.error as Error).message} +

+ )} +
+ ); +} diff --git a/frontend/src/components/dex/LpPositionBar.tsx b/frontend/src/components/dex/LpPositionBar.tsx index 5e8c34eb3..1ba31a302 100644 --- a/frontend/src/components/dex/LpPositionBar.tsx +++ b/frontend/src/components/dex/LpPositionBar.tsx @@ -2,15 +2,7 @@ import { useMemo } from "react"; import { feeAmount, lpStateStyle, readLpPosition, type LpPosition } from "./lp-position"; import { type ExecutorInfo } from "@/lib/api"; -import { formatPnl, isExecutorActive, pnlColor } from "@/lib/formatters"; - -/** Prices here span memecoin to majors, so significant digits, not decimals. */ -function fmtPrice(price: number | null): string { - if (price === null || !Number.isFinite(price) || price <= 0) return "—"; - if (price >= 1000) return price.toFixed(2); - if (price >= 1) return price.toFixed(4); - return price.toPrecision(5); -} +import { formatPnl, formatPriceSig, isExecutorActive, pnlColor } from "@/lib/formatters"; /** * Where the price sits inside the range, as a 0–1 fraction, or `null`. @@ -91,7 +83,7 @@ export function LpPositionBar({ {/* Range, with the price's place inside it */} - {fmtPrice(pos.lowerPrice)} + {formatPriceSig(pos.lowerPrice)} {fraction !== null && ( )} - {fmtPrice(pos.upperPrice)} + {formatPriceSig(pos.upperPrice)} diff --git a/frontend/src/components/dex/PoolBrowser.tsx b/frontend/src/components/dex/PoolBrowser.tsx index 5da202c38..894fb92bf 100644 --- a/frontend/src/components/dex/PoolBrowser.tsx +++ b/frontend/src/components/dex/PoolBrowser.tsx @@ -11,6 +11,8 @@ import { useNavigate } from "react-router-dom"; import type { PoolSummary } from "@/lib/api"; import { useDexFavorites } from "@/lib/dexFavorites"; +import { num, pct, usdCompact } from "./format"; + type SortKey = | "name" | "dex_id" @@ -21,37 +23,9 @@ type SortKey = | "apy"; type SortDir = "asc" | "desc"; -/** - * Both upstreams answer numbers as strings (and GeckoTerminal answers pandas NaN - * for a missing figure). The backend settles that before it serializes, so this - * is the second line of defence rather than the first — but it is the one that - * decides whether a stray string blanks one cell or takes down the whole page. - */ -function num(v: unknown): number | null { - if (v === null || v === undefined || v === "") return null; - const n = typeof v === "number" ? v : parseFloat(String(v)); - return Number.isFinite(n) ? n : null; -} - -function usd(v: unknown): string { - const n = num(v); - if (!n) return "—"; - if (n >= 1e9) return `$${(n / 1e9).toFixed(2)}B`; - if (n >= 1e6) return `$${(n / 1e6).toFixed(2)}M`; - if (n >= 1e3) return `$${(n / 1e3).toFixed(1)}K`; - return `$${n.toFixed(2)}`; -} - -function pct(v: unknown): string { - const n = num(v); - if (n === null) return "—"; - if (Math.abs(n) >= 10_000) return `${(n / 1000).toFixed(0)}K%`; - return `${n.toFixed(2)}%`; -} - -/** Sorting compares numbers, and a missing figure sorts last in either direction. */ -function sortValue(p: PoolSummary, key: SortKey): number { - return num(p[key as keyof PoolSummary]) ?? -Infinity; +/** Null-preserving so the comparator can push missing figures last either way. */ +function sortValue(p: PoolSummary, key: SortKey): number | null { + return num(p[key as keyof PoolSummary]); } /** Orca's Gateway listing carries no yield at all; the backend derives one. */ @@ -158,7 +132,17 @@ export function PoolBrowser({ let cmp: number; if (sortKey === "name") cmp = pairLabel(a).localeCompare(pairLabel(b)); else if (sortKey === "dex_id") cmp = a.dex_id.localeCompare(b.dex_id); - else cmp = sortValue(a, sortKey) - sortValue(b, sortKey); + else { + const av = sortValue(a, sortKey); + const bv = sortValue(b, sortKey); + // A missing figure sorts last in either direction, so it sits outside + // the direction flip below. + if (av === null || bv === null) { + if (av === bv) return 0; + return av === null ? 1 : -1; + } + cmp = av - bv; + } return sortDir === "asc" ? cmp : -cmp; }); return rows; @@ -345,10 +329,10 @@ export function PoolBrowser({ {p.dex_id} - {usd(p.reserve_usd)} + {usdCompact(p.reserve_usd)} - {usd(p.volume_24h)} + {usdCompact(p.volume_24h)} void; }) { const [open, setOpen] = useState(false); - const ref = useRef(null); - - useEffect(() => { - const handler = (e: MouseEvent) => { - if (ref.current && !ref.current.contains(e.target as Node)) setOpen(false); - }; - const keyHandler = (e: KeyboardEvent) => { - if (e.key === "Escape") setOpen(false); - }; - document.addEventListener("mousedown", handler); - document.addEventListener("keydown", keyHandler); - return () => { - document.removeEventListener("mousedown", handler); - document.removeEventListener("keydown", keyHandler); - }; - }, []); + const [anchor, setAnchor] = useState(null); + const close = useCallback(() => setOpen(false), []); // One chain is not a choice — render it as a label rather than a dead dropdown. if (chains.length <= 1) return null; @@ -68,8 +55,9 @@ function ChainSelect({ const current = chains.find((c) => c.network_id === network); return ( -
+ <> - {open && ( -
- {chains.map((c) => ( - - ))} -
- )} -
+ + {chains.map((c) => ( + + ))} + + ); } @@ -126,23 +112,8 @@ function DexFilter({ }) { const [open, setOpen] = useState(false); const [filter, setFilter] = useState(""); - const ref = useRef(null); - - useEffect(() => { - const handler = (e: MouseEvent) => { - if (ref.current && !ref.current.contains(e.target as Node)) - setOpen(false); - }; - const keyHandler = (e: KeyboardEvent) => { - if (e.key === "Escape") setOpen(false); - }; - document.addEventListener("mousedown", handler); - document.addEventListener("keydown", keyHandler); - return () => { - document.removeEventListener("mousedown", handler); - document.removeEventListener("keydown", keyHandler); - }; - }, []); + const [anchor, setAnchor] = useState(null); + const close = useCallback(() => setOpen(false), []); const shown = useMemo(() => { const needle = filter.trim().toLowerCase(); @@ -171,8 +142,9 @@ function DexFilter({ : `${selected.length} DEXes`; return ( -
+ <> - {open && ( -
-
- setFilter(e.target.value)} - placeholder="Filter venues…" - autoFocus - spellCheck={false} - className="w-full rounded-md border border-[var(--color-border)] bg-[var(--color-bg)] px-2 py-1 text-xs text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]" - /> -
-
- {shown.map((v) => { + +
+ setFilter(e.target.value)} + placeholder="Filter venues…" + autoFocus + spellCheck={false} + className="w-full rounded-md border border-[var(--color-border)] bg-[var(--color-bg)] px-2 py-1 text-xs text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]" + /> +
+
+ {shown.map((v) => { const active = selected.includes(v.id); - return ( - - ); - })} - {!shown.length && ( -
- No venue matches. -
- )} -
+ {active && } + + {v.name} + + ); + })} + {!shown.length && ( +
+ No venue matches. +
+ )}
- )} -
+ + ); } diff --git a/frontend/src/components/dex/PoolStats.tsx b/frontend/src/components/dex/PoolStats.tsx index f82aef4d0..48c23baa3 100644 --- a/frontend/src/components/dex/PoolStats.tsx +++ b/frontend/src/components/dex/PoolStats.tsx @@ -2,6 +2,20 @@ import { Check, Copy, ExternalLink } from "lucide-react"; import { useState } from "react"; import type { PoolSummary } from "@/lib/api"; +import { copyText } from "@/lib/clipboard"; + +import { num, pct, usdCompact } from "./format"; + +/** GeckoTerminal chain ids → the slugs the DEX web apps route on. */ +const UNISWAP_CHAINS: Record = { + eth: "ethereum", + bsc: "bnb", + polygon_pos: "polygon", +}; + +const PANCAKE_CHAINS: Record = { + arbitrum: "arb", +}; /** The DEX web app URL for a pool — the same mapping Telegram's /lp offers. */ function poolUrl(pool: PoolSummary): string { @@ -11,24 +25,20 @@ function poolUrl(pool: PoolSummary): string { return `https://app.meteora.ag/dlmm/${addr}?referrer=hummingbot`; if (dex.startsWith("raydium")) return `https://raydium.io/clmm/pool/${addr}`; if (dex.startsWith("orca")) return `https://www.orca.so/pools/${addr}`; + if (dex.startsWith("pumpswap") && pool.base_token_address) + return `https://pump.fun/coin/${pool.base_token_address}`; + if (dex.startsWith("uniswap")) { + const chain = UNISWAP_CHAINS[pool.gecko_network] ?? pool.gecko_network; + return `https://app.uniswap.org/explore/pools/${chain}/${addr}`; + } + if (dex.startsWith("pancakeswap")) { + const chain = PANCAKE_CHAINS[pool.gecko_network] ?? pool.gecko_network; + return `https://pancakeswap.finance/liquidity/pool/${chain}/${addr}`; + } // GeckoTerminal indexes every pool it listed, so it is the universal fallback. return `https://www.geckoterminal.com/${pool.gecko_network}/pools/${addr}`; } -function num(v: unknown): number { - const n = typeof v === "string" ? parseFloat(v) : (v as number); - return Number.isFinite(n) ? n : 0; -} - -function usd(v: unknown): string { - const n = num(v); - if (!n) return "—"; - if (n >= 1e9) return `$${(n / 1e9).toFixed(2)}B`; - if (n >= 1e6) return `$${(n / 1e6).toFixed(2)}M`; - if (n >= 1e3) return `$${(n / 1e3).toFixed(1)}K`; - return `$${n.toFixed(2)}`; -} - function Stat({ label, value, tone }: { label: string; value: string; tone?: string }) { return (
@@ -42,27 +52,56 @@ function Stat({ label, value, tone }: { label: string; value: string; tone?: str ); } -/** The pool's identity and market numbers, above the chart. */ -export function PoolStats({ pool }: { pool: PoolSummary }) { +/** The shortened pool address with copy and open-on-DEX actions. */ +export function PoolAddress({ pool }: { pool: PoolSummary }) { const [copied, setCopied] = useState(false); - const change = pool.price_change_24h; - const copy = () => { - navigator.clipboard.writeText(pool.address); + const copy = async () => { + if (!(await copyText(pool.address))) return; setCopied(true); setTimeout(() => setCopied(false), 1500); }; + return ( + + + {pool.address.slice(0, 6)}…{pool.address.slice(-4)} + + + + + + + ); +} + +/** The pool's identity and market numbers, above the chart. */ +export function PoolStats({ pool, network }: { pool: PoolSummary; network?: string }) { + const change = num(pool.price_change_24h); + return (
+ {network && } - - + + = 0 ? "text-[var(--color-green)]" @@ -70,7 +109,7 @@ export function PoolStats({ pool }: { pool: PoolSummary }) { } /> {pool.apr !== null && pool.apr !== undefined && ( - + )} {pool.bin_step !== null && pool.bin_step !== undefined && ( @@ -79,31 +118,6 @@ export function PoolStats({ pool }: { pool: PoolSummary }) { pool.base_fee_percentage !== undefined && ( )} - -
- - {pool.address.slice(0, 6)}…{pool.address.slice(-4)} - - - - - -
); } diff --git a/frontend/src/components/dex/format.ts b/frontend/src/components/dex/format.ts new file mode 100644 index 000000000..cb20472b5 --- /dev/null +++ b/frontend/src/components/dex/format.ts @@ -0,0 +1,46 @@ +/** + * The one copy of the unknown-input number/USD/percent formatters the DEX + * components share. PoolBrowser, PoolStats and lp-position each grew a private + * copy of these that silently diverged (zero-vs-missing semantics, percent + * capping); this module is the single definition they all import. + */ + +/** + * Both upstreams answer numbers as strings (and GeckoTerminal answers pandas NaN + * for a missing figure). The backend settles that before it serializes, so this + * is the second line of defence rather than the first — but it is the one that + * decides whether a stray string blanks one cell or takes down the whole page. + * + * Null-preserving: a missing figure stays `null` so callers can tell "no data" + * from a genuine zero. + */ +export function num(v: unknown): number | null { + if (v === null || v === undefined || v === "") return null; + const n = typeof v === "number" ? v : parseFloat(String(v)); + return Number.isFinite(n) ? n : null; +} + +/** + * Compact dollars with a billions tier for pool TVL and volume figures. + * Only a *missing* figure renders as an em-dash; a genuine `$0` renders as + * `$0.00` — zero TVL is a fact about the pool, not absent data. + */ +export function usdCompact(v: unknown): string { + const n = num(v); + if (n === null) return "—"; + if (n >= 1e9) return `$${(n / 1e9).toFixed(2)}B`; + if (n >= 1e6) return `$${(n / 1e6).toFixed(2)}M`; + if (n >= 1e3) return `$${(n / 1e3).toFixed(1)}K`; + return `$${n.toFixed(2)}`; +} + +/** + * Percent with a K% cap: a 25000% memecoin move renders as "25K%" instead of + * stretching the column with "25000.00%". Em-dash for a missing figure. + */ +export function pct(v: unknown): string { + const n = num(v); + if (n === null) return "—"; + if (Math.abs(n) >= 10_000) return `${(n / 1000).toFixed(0)}K%`; + return `${n.toFixed(2)}%`; +} diff --git a/frontend/src/components/dex/lp-position.ts b/frontend/src/components/dex/lp-position.ts index b55749d11..f97def4cc 100644 --- a/frontend/src/components/dex/lp-position.ts +++ b/frontend/src/components/dex/lp-position.ts @@ -1,6 +1,8 @@ import { type ExecutorInfo } from "@/lib/api"; import { formatUsd } from "@/lib/formatters"; +import { num } from "./format"; + /** LP executors are the only ones the DEX pages can route to a pool. */ export const LP_EXECUTOR_TYPE = "lp_executor"; @@ -22,12 +24,6 @@ export const LP_REFRESH_MS = 20_000; */ export const RECENT_LP_EXECUTORS = 200; -function num(v: unknown): number | null { - if (v === null || v === undefined || v === "") return null; - const n = typeof v === "number" ? v : parseFloat(String(v)); - return Number.isFinite(n) ? n : null; -} - function str(v: unknown): string { return typeof v === "string" ? v : ""; } diff --git a/frontend/src/components/executor/ExecutorSuccessModal.tsx b/frontend/src/components/executor/ExecutorSuccessModal.tsx new file mode 100644 index 000000000..6e8f3167d --- /dev/null +++ b/frontend/src/components/executor/ExecutorSuccessModal.tsx @@ -0,0 +1,117 @@ +import { Check, CheckCircle, Copy, X } from "lucide-react"; +import { useEffect, useState, type ReactNode } from "react"; + +import { copyText } from "@/lib/clipboard"; + +interface ExecutorSuccessModalProps { + executorId: string; + title: string; + subtitle?: string; + /** Extra rows rendered inside the details box, after the Executor ID row. */ + details?: ReactNode; + /** Label for the primary action button (default "Continue"). */ + primaryLabel?: string; + /** Handler for the primary action button (defaults to onClose). */ + onPrimary?: () => void; + onClose: () => void; +} + +/** + * Success dialog shown after creating an executor (order, LP position, grid…). + * Shared by DexPool, CreateExecutor and CreateGridExecutor so the overlay, + * card styling and copy-ID behavior stay consistent. + */ +export function ExecutorSuccessModal({ + executorId, + title, + subtitle, + details, + primaryLabel = "Continue", + onPrimary, + onClose, +}: ExecutorSuccessModalProps) { + const [copied, setCopied] = useState(false); + + useEffect(() => { + if (!copied) return; + const t = setTimeout(() => setCopied(false), 1500); + return () => clearTimeout(t); + }, [copied]); + + return ( +
+
+ {/* Close button */} + + + {/* Icon */} +
+
+ +
+
+ + {/* Title */} +

+ {title} +

+ {subtitle && ( +

+ {subtitle} +

+ )} + + {/* Details */} +
+
+ Executor ID +
+ + {executorId.slice(0, 12)}… + + +
+
+ {details} +
+ + {/* Action */} + +
+
+ ); +} + +/** + * Bottom-right error toast for executor create mutations. Rendered inside a + * relatively-positioned page container. + */ +export function ErrorToast({ message }: { message: string }) { + return ( +
+ {message} +
+ ); +} diff --git a/frontend/src/components/executor/LPConfigPanel.tsx b/frontend/src/components/executor/LPConfigPanel.tsx index 4c62c6fcb..a3a8b4a82 100644 --- a/frontend/src/components/executor/LPConfigPanel.tsx +++ b/frontend/src/components/executor/LPConfigPanel.tsx @@ -78,6 +78,13 @@ interface Props { pair?: string; pool?: DexPoolInfo; poolFetching?: boolean; + /** + * The pool's DLMM bin step in basis points, when the caller knows it. Only the + * pool workspace does — the resolve-by-pair endpoint the create page uses does + * not report one — and it is what turns a too-wide range into a warning here + * rather than a Gateway error at open time. + */ + binStep?: number | null; /** Wallet balance behind the percentage presets; `null` when it is unknown. */ baseAvailable?: number | null; quoteAvailable?: number | null; @@ -87,6 +94,12 @@ interface Props { */ baseSymbol?: string; quoteSymbol?: string; + /** + * Set when the page itself pins the pool (the DEX pool workspace). The panel + * then just states the address — resolution status, an editable address and a + * provider picker would only second-guess a decision already made. + */ + lockedPoolAddress?: string; } export function LPConfigPanel({ @@ -97,10 +110,12 @@ export function LPConfigPanel({ pair, pool, poolFetching, + binStep, baseAvailable = null, quoteAvailable = null, baseSymbol, quoteSymbol, + lockedPoolAddress, }: Props) { const d = dispatch as FieldDispatch; @@ -120,75 +135,86 @@ export function LPConfigPanel({ } }, [price, unanchored]); // eslint-disable-line react-hooks/exhaustive-deps - const warnings = rangeWarnings(state, price); + // Only DLMM pools carry a bin step, so passing it through unconditionally is + // enough to scope the bin-count warning to the pools the cap applies to. + const warnings = rangeWarnings(state, price, binStep); const unsupported = !!pool && !pool.lp_supported; return (
{/* Resolved pool */} -
- Pool - {poolFetching && !pool ? ( -

Resolving pool…

- ) : unsupported ? ( -
-

- - - The deepest pool for this pair is on{" "} - {pool?.dex_id ?? "an unknown venue"}, which - is not a CLMM venue. Enter a pool address and provider by hand. - -

-
- ) : pool?.pool_address ? ( -
- - {pool.dex_id} - - - {truncateAddress(pool.pool_address)} - - {pool.current_price != null && ( - - {formatPoolPrice(pool.current_price)} {quoteAsset} + {lockedPoolAddress ? ( +

+ {truncateAddress(lockedPoolAddress)} +

+ ) : ( +
+ Pool + {poolFetching && !pool ? ( +

Resolving pool…

+ ) : unsupported ? ( +
+

+ + + The deepest pool for this pair is on{" "} + {pool?.dex_id ?? "an unknown venue"}, which + is not a CLMM venue. Enter a pool address and provider by hand. + +

+
+ ) : pool?.pool_address ? ( +
+ + {pool.dex_id} - )} - {state.poolTouched && ( - - overridden + + {truncateAddress(pool.pool_address)} - )} -
- ) : ( -

- No pool found for this pair. Enter one by hand. -

- )} + {pool.current_price != null && ( + + {formatPoolPrice(pool.current_price)} {quoteAsset} + + )} + {state.poolTouched && ( + + overridden + + )} +
+ ) : ( +

+ No pool found for this pair. Enter one by hand. +

+ )} -
- - - d({ type: "SET_FIELD", field: "pool_address", value: e.target.value.trim() }) - } - placeholder="Pool contract address" - spellCheck={false} - className="w-full rounded border border-[var(--color-border)] bg-[var(--color-bg)] px-2.5 py-1.5 font-mono text-[11px] text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]/40 focus:border-[var(--color-primary)] focus:outline-none" +
+ + + d({ type: "SET_FIELD", field: "pool_address", value: e.target.value.trim() }) + } + placeholder="Pool contract address" + spellCheck={false} + className="w-full rounded border border-[var(--color-border)] bg-[var(--color-bg)] px-2.5 py-1.5 font-mono text-[11px] text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]/40 focus:border-[var(--color-primary)] focus:outline-none" + /> +
+
- -
+ )} {/* Side */}
diff --git a/frontend/src/components/executor/OrderConfigPanel.tsx b/frontend/src/components/executor/OrderConfigPanel.tsx index 2a9b3aab4..d6c707dc4 100644 --- a/frontend/src/components/executor/OrderConfigPanel.tsx +++ b/frontend/src/components/executor/OrderConfigPanel.tsx @@ -251,8 +251,9 @@ export function OrderConfigPanel({ return (
- {/* Direction */} - + {/* Direction — Buy/Sell on spot: Long/Short would suggest a perp, and a + "short" without tokens to sell is not a thing a swap can do. */} + {/* Order Config */}
@@ -280,6 +281,8 @@ export function OrderConfigPanel({ field="execution_strategy" dispatch={d} options={options} + // A DEX only swaps at market; a one-option dropdown reads as broken. + disabled={options.length <= 1} /> {!isSpot && ( diff --git a/frontend/src/components/executor/fields.tsx b/frontend/src/components/executor/fields.tsx index d4d959c0b..d5a4c3f33 100644 --- a/frontend/src/components/executor/fields.tsx +++ b/frontend/src/components/executor/fields.tsx @@ -227,12 +227,15 @@ export function SelectField({ field, dispatch, options, + disabled = false, }: { label: string; value: number | string; field: string; dispatch: FieldDispatch; options: { value: number | string; label: string }[]; + /** A select with nothing to select reads as broken unless it says so. */ + disabled?: boolean; }) { const [open, setOpen] = useState(false); const containerRef = useRef(null); @@ -265,13 +268,20 @@ export function SelectField({ aria-label={label} aria-expanded={open} aria-haspopup="listbox" + disabled={disabled} onClick={() => setOpen(!open)} - className="flex w-full items-center justify-between rounded-lg border border-[var(--color-border)] bg-[var(--color-surface)] px-2.5 py-1.5 text-xs text-[var(--color-text)] transition-colors hover:bg-[var(--color-surface-hover)]" + className={`flex w-full items-center justify-between rounded-lg border border-[var(--color-border)] px-2.5 py-1.5 text-xs transition-colors ${ + disabled + ? "bg-[var(--color-bg)] text-[var(--color-text-muted)] opacity-60" + : "bg-[var(--color-surface)] text-[var(--color-text)] hover:bg-[var(--color-surface-hover)]" + }`} > {selected?.label} - + {!disabled && ( + + )} - {open && ( + {open && !disabled && (
{options.map((opt) => { const isActive = opt.value === value; @@ -533,7 +543,16 @@ export function LeverageField({ -export function SideSelector({ side, dispatch }: { side: 1 | 2; dispatch: FieldDispatch }) { +export function SideSelector({ + side, + dispatch, + isSpot = false, +}: { + side: 1 | 2; + dispatch: FieldDispatch; + /** Spot has no position to be long or short of — the sides are Buy and Sell. */ + isSpot?: boolean; +}) { return (
Direction @@ -549,7 +568,7 @@ export function SideSelector({ side, dispatch }: { side: 1 | 2; dispatch: FieldD }`} style={side === opt.value ? { backgroundColor: opt.color } : undefined} > - {opt.label} + {isSpot ? (opt.value === 1 ? "BUY" : "SELL") : opt.label} ))}
diff --git a/frontend/src/components/executor/lp-config.ts b/frontend/src/components/executor/lp-config.ts index 5b966af6c..247a83a43 100644 --- a/frontend/src/components/executor/lp-config.ts +++ b/frontend/src/components/executor/lp-config.ts @@ -228,12 +228,40 @@ export function useLpValidation(state: LPState): ExecutorValidation { }, [state]); } +/** + * Bins a Meteora DLMM position of this width would span. + * + * Each bin is a fixed geometric step of `1 + bin_step/10000`, so the count is the + * log of the price ratio over the log of that step. Null when the pool is not a + * DLMM (only those report a bin step) or the bounds are not a real range. + */ +export function dlmmBinSpan( + lower: number, + upper: number, + binStep: number | null | undefined, +): number | null { + if (!binStep || binStep <= 0) return null; + if (!(lower > 0) || !(upper > lower)) return null; + return Math.ceil(Math.log(upper / lower) / Math.log(1 + binStep / 10000)); +} + +/** + * What one Meteora DLMM position can hold. A wider range is not rejected by the + * form — Gateway is the authority and its own cap can move — but it fails at + * open time, and until now the only trace of that was a line in the gateway log. + */ +export const DLMM_MAX_BINS = 69; + /** * Things the schema permits but that are rarely meant. Kept apart from validation * because none of them should block a create — a deliberately odd position is * still a position. */ -export function rangeWarnings(state: LPState, price: number | null): string[] { +export function rangeWarnings( + state: LPState, + price: number | null, + binStep?: number | null, +): string[] { const warnings: string[] = []; const bothAmounts = state.base_amount > 0 && state.quote_amount > 0; @@ -257,6 +285,13 @@ export function rangeWarnings(state: LPState, price: number | null): string[] { if (state.lower_limit_price > 0 && state.lower_limit_price >= state.lower_price) { warnings.push("Lower limit sits inside the range"); } + + const bins = dlmmBinSpan(state.lower_price, state.upper_price, binStep); + if (bins !== null && bins > DLMM_MAX_BINS) { + warnings.push( + `Range spans ~${bins} bins; Meteora opens at most ${DLMM_MAX_BINS} per position — narrow it or the open will fail`, + ); + } return warnings; } @@ -278,6 +313,15 @@ const SLOT_FIELD: Record = { limit2: "lower_limit_price", }; +// The slots are named for the grid executor; on an LP range they are prices, +// not a direction of travel, so the chart is told what they mean here. +const LP_LINE_LABELS: Partial> = { + start: "Upper", + end: "Lower", + limit: "Upper limit", + limit2: "Lower limit", +}; + export function isMeteoraProvider(provider: string): boolean { return provider.toLowerCase().startsWith("meteora/"); } @@ -314,6 +358,7 @@ export function useLpConfig( side: state.side === LP_SIDE_SELL ? 2 : 1, minSpread: 0, activePickField: PICK_SLOT[state.activePickField ?? ""] ?? null, + lineLabels: LP_LINE_LABELS, extraLines: state.lower_limit_price > 0 ? [ diff --git a/frontend/src/components/executor/types.ts b/frontend/src/components/executor/types.ts index ea549dadc..aadb46804 100644 --- a/frontend/src/components/executor/types.ts +++ b/frontend/src/components/executor/types.ts @@ -33,6 +33,13 @@ export interface ChartPriceMapping { minSpread: number; activePickField: PickSlot | null; extraLines?: ExtraLine[]; + /** + * What the chart calls each slot's line and pick hint. The slots are named + * for the grid executor ("Start", "End", "Limit"); a panel that rides them + * with different semantics — LP's upper/lower bounds — relabels them here + * rather than letting the chart mislabel its prices. + */ + lineLabels?: Partial>; } export interface ExecutorValidation { diff --git a/frontend/src/components/layout/AppShell.tsx b/frontend/src/components/layout/AppShell.tsx index a454f4e83..8a43fefc6 100644 --- a/frontend/src/components/layout/AppShell.tsx +++ b/frontend/src/components/layout/AppShell.tsx @@ -18,12 +18,14 @@ import { NavLink, Outlet, useLocation, useNavigate } from "react-router-dom"; import { ConnectKeysOverlay } from "@/components/ConnectKeysOverlay"; import { ErrorBoundary } from "@/components/ErrorBoundary"; import { ReportIssueDialog } from "@/components/ReportIssueDialog"; +import { TelemetryConsentBanner } from "@/components/TelemetryConsentBanner"; import { ChatProvider } from "@/hooks/useChat"; import { useCredentials } from "@/hooks/useCredentials"; import { usePrefetchData } from "@/hooks/usePrefetchData"; import { useServer } from "@/hooks/useServer"; import { useTheme } from "@/hooks/useTheme"; import { CurrencySelector } from "./CurrencySelector"; +import { NotificationBell } from "./NotificationBell"; import { ServerSelector } from "./ServerSelector"; const NAV_ITEMS = [ @@ -128,6 +130,12 @@ function AppShellBody() {
+ {/* Inside the ChatProvider on purpose: the bell reads the same + react-query cache the chat socket pushes live notifications + into, and that socket is what makes it update without a + reload (FEAT-048). */} + +
+ {/* Asks once, for an install that has never answered (FEAT-023). Renders + nothing for everyone else, including on the chat workspace, which owns + its own scrolling — hence outside `main` rather than inside it. */} + + {/* Main content */}
= { + delegation: Workflow, + routine: Zap, + agent: Bot, + system: Terminal, +}; + +/** A notification is one line in a dropdown; anything longer is a wall of text. */ +const MAX_LINES = 3; + +function preview(text: string): string { + const lines = text.split("\n").filter((l) => l.trim()); + const head = lines.slice(0, MAX_LINES).join("\n"); + return lines.length > MAX_LINES ? `${head}…` : head; +} + +/** + * The dashboard's notification surface (FEAT-048). + * + * Everything Condor says when a background task finishes — a delegation + * completing, a routine run ending, an agent's `send_notification`, the boot + * notice — used to go to Telegram and nowhere else. This is where it lands for + * a user who only has the browser open. + * + * Two sources, one cache. The history comes from `GET /notifications` on mount; + * live arrivals are written into the same react-query key by the chat socket's + * `notification` handler, which is why a notice that arrived while the tab was + * open and one that arrived an hour ago are indistinguishable here. That is the + * whole point: the store, not the socket, is what guarantees it is seen. + */ +export function NotificationBell() { + const { token } = useAuth(); + const [open, setOpen] = useState(false); + const ref = useRef(null); + const navigate = useNavigate(); + const queryClient = useQueryClient(); + + const { data } = useQuery({ + queryKey: NOTIFICATIONS_KEY, + queryFn: () => api.getNotifications(50), + // A push keeps this fresh while a tab is open; the refetch is only for the + // socket being down, so it is slow on purpose. + staleTime: 60_000, + enabled: !!token, + }); + + const items = data?.items ?? []; + const unread = data?.unread ?? 0; + + const markRead = useMutation({ + mutationFn: (ids?: string[]) => api.markNotificationsRead(ids), + onSuccess: (res) => { + queryClient.setQueryData(NOTIFICATIONS_KEY, (prev) => + prev + ? { items: prev.items.map((n) => ({ ...n, read: true })), unread: res.unread } + : prev, + ); + }, + }); + + useEffect(() => { + const handler = (e: MouseEvent) => { + if (ref.current && !ref.current.contains(e.target as Node)) setOpen(false); + }; + const keyHandler = (e: KeyboardEvent) => { + if (e.key === "Escape") setOpen(false); + }; + document.addEventListener("mousedown", handler); + document.addEventListener("keydown", keyHandler); + return () => { + document.removeEventListener("mousedown", handler); + document.removeEventListener("keydown", keyHandler); + }; + }, []); + + const toggle = () => { + const next = !open; + setOpen(next); + // Opening the list *is* reading it. The badge is there to pull the user in, + // so leaving it lit after they looked would train them to ignore it. + if (next && unread > 0) markRead.mutate(undefined); + }; + + const openItem = (n: AppNotification) => { + if (!n.link) return; + setOpen(false); + navigate(n.link); + }; + + return ( +
+ + + {open && ( +
+
+ + Notifications + + {items.length > 0 && ( + + )} +
+ + {items.length === 0 && ( +
+ Nothing yet. Finished tasks show up here. +
+ )} + + {items.map((n) => { + const Icon = KIND_ICON[n.kind] ?? Bell; + return ( + + ); + })} +
+ )} +
+ ); +} diff --git a/frontend/src/components/layout/ServerSelector.tsx b/frontend/src/components/layout/ServerSelector.tsx index 84b384d69..6a9ee1256 100644 --- a/frontend/src/components/layout/ServerSelector.tsx +++ b/frontend/src/components/layout/ServerSelector.tsx @@ -1,5 +1,5 @@ import { useQuery } from "@tanstack/react-query"; -import { ChevronDown, Circle, Server } from "lucide-react"; +import { ChevronDown, Circle, Server, Star } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; import { useServer } from "@/hooks/useServer"; @@ -27,12 +27,22 @@ export function ServerSelector() { [JSON.stringify(onlineServers.map((s) => s.name))], ); - // Auto-select first online server only when no server is saved yet + // The server the backend calls this user's default — the same `chat_defaults` + // entry Telegram sets. A browser with nothing saved opens on it, so setting a + // default in either surface is honoured by the other. + const defaultOnlineName = useMemo( + () => onlineServers.find((s) => s.is_default)?.name ?? null, + // eslint-disable-next-line react-hooks/exhaustive-deps + [JSON.stringify(onlineServers.map((s) => [s.name, s.is_default]))], + ); + + // Auto-select only when no server is saved yet: the user's default if it is + // up, otherwise the first server that is. useEffect(() => { if (!server && onlineServerNames.length > 0) { - setServer(onlineServerNames[0]); + setServer(defaultOnlineName ?? onlineServerNames[0]); } - }, [server, onlineServerNames, setServer]); + }, [server, onlineServerNames, defaultOnlineName, setServer]); // Close on outside click or Escape useEffect(() => { @@ -90,6 +100,12 @@ export function ServerSelector() { > {s.name} + {s.is_default && ( + + )} {s.name === server && ( Active )} diff --git a/frontend/src/components/settings/GatewaySettings.tsx b/frontend/src/components/settings/GatewaySettings.tsx index b6fb42bd6..7a42c1c70 100644 --- a/frontend/src/components/settings/GatewaySettings.tsx +++ b/frontend/src/components/settings/GatewaySettings.tsx @@ -99,6 +99,32 @@ function NetworkConfigEditor({ server, networkId }: { server: string; networkId: ([key, value]) => value !== String(config[key] ?? ""), ); + // Non-owners get server-side masked values (SEC-197) and their saves are + // refused anyway (SEC-166), so render a read-only view instead of a form + // that could only fail — or worse, round-trip a masked value into a save. + const readOnly = !isOwner || data.redacted === true; + + if (readOnly) { + return ( +
+ {editableEntries.map(([key, value]) => ( +
+ + + {String(value ?? "")} + +
+ ))} +

{OWNER_ONLY_HINT}

+
+ ); + } + const save = () => { const updates: Record = {}; for (const [key, value] of changed) { diff --git a/frontend/src/components/settings/ImportGatewayWallet.tsx b/frontend/src/components/settings/ImportGatewayWallet.tsx index 75afcc057..af7583f63 100644 --- a/frontend/src/components/settings/ImportGatewayWallet.tsx +++ b/frontend/src/components/settings/ImportGatewayWallet.tsx @@ -114,7 +114,13 @@ export function ImportGatewayWallet({ setKey(e.target.value)} disabled={saving} diff --git a/frontend/src/components/settings/ServersSettings.tsx b/frontend/src/components/settings/ServersSettings.tsx index 7331e962b..5eea5f873 100644 --- a/frontend/src/components/settings/ServersSettings.tsx +++ b/frontend/src/components/settings/ServersSettings.tsx @@ -10,6 +10,7 @@ import { } from "lucide-react"; import { useState } from "react"; +import { useServer } from "@/hooks/useServer"; import { type ServerInfo, api } from "@/lib/api"; interface ServerForm { @@ -24,6 +25,7 @@ const EMPTY_FORM: ServerForm = { name: "", host: "", port: 8000, username: "", p export function ServersSettings() { const qc = useQueryClient(); + const { server: activeServer, setServer } = useServer(); const [adding, setAdding] = useState(false); const [editing, setEditing] = useState(null); const [form, setForm] = useState(EMPTY_FORM); @@ -54,9 +56,16 @@ export function ServersSettings() { onSuccess: () => { invalidate(); setConfirmDelete(null); }, }); + // Setting the default is two things at once, because the button reads as one: + // it writes the `chat_defaults` entry Telegram also uses, and it points this + // browser at that server. Without the second half the click had no visible + // effect at all — the star was never filled and the header never moved. const defaultMut = useMutation({ mutationFn: (name: string) => api.setDefaultServer(name), - onSuccess: invalidate, + onSuccess: (_data, name) => { + if (name !== activeServer) setServer(name); + invalidate(); + }, }); const startEdit = (s: ServerInfo) => { @@ -204,6 +213,11 @@ export function ServersSettings() { {s.permission} + {s.is_default && ( + + Default + + )}
{s.host}:{s.port} @@ -214,10 +228,21 @@ export function ServersSettings() {
{isOwner(s) && ( <> diff --git a/frontend/src/components/settings/TelemetrySettings.tsx b/frontend/src/components/settings/TelemetrySettings.tsx new file mode 100644 index 000000000..2d9912ec7 --- /dev/null +++ b/frontend/src/components/settings/TelemetrySettings.tsx @@ -0,0 +1,125 @@ +import { BarChart3, Check, Loader2, Lock, ShieldCheck } from "lucide-react"; + +import { useSetTelemetryLevel, useTelemetry } from "@/hooks/useTelemetry"; +import type { TelemetryLevel } from "@/lib/api"; + +/** + * What this install shares, and — for the admin — the switch that changes it. + * + * Readable by every seat on purpose: anyone using an install should be able to + * see what it reports, even if only the admin may answer. It is also the one + * surface a local-mode install has, since there is no Telegram prompt there. + */ +export function TelemetrySettings() { + const { data, isLoading } = useTelemetry(); + const mutation = useSetTelemetryLevel(); + + if (isLoading || !data) { + return ( +
+ +
+ ); + } + + const { disclosure, level, consent, env_overridden, can_change } = data; + const locked = env_overridden || !can_change; + // At `unknown` no option is the answer yet, so none is shown as chosen — + // the install is at the ping floor, but nobody has said so. + const chosen = consent === "granted" ? level : null; + + return ( +
+
+

+ + {disclosure.headline} +

+
+

{disclosure.always_on}

+

{disclosure.optional}

+
+
+ +
+

+ Never sent +

+
    + {disclosure.never.map((item) => ( +
  • + {item} +
    • + ))} +
+
+ +
+

+ This install +

+
+ {disclosure.options.map((option) => { + const active = chosen === option.level; + return ( + + ); + })} +
+ + {env_overridden && ( +

+ + + CONDOR_TELEMETRY is set in this install's environment + and overrides anything chosen here. Effective level:{" "} + {level}. + +

+ )} + {!env_overridden && !can_change && ( +

+ + + Telemetry is install-wide, so only the admin can change it. + Effective level: {level}. + +

+ )} + {mutation.isError && ( +

+ Could not save that. {(mutation.error as Error).message} +

+ )} +
+ +

{disclosure.doc}

+
+ ); +} diff --git a/frontend/src/components/trade/TradeBottomPane.tsx b/frontend/src/components/trade/TradeBottomPane.tsx index 0f2d9104e..10702608e 100644 --- a/frontend/src/components/trade/TradeBottomPane.tsx +++ b/frontend/src/components/trade/TradeBottomPane.tsx @@ -1,14 +1,16 @@ import { useState, useEffect, useCallback, useRef, useMemo } from "react"; import { createPortal } from "react-dom"; -import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; +import { useMutation, useQueryClient } from "@tanstack/react-query"; import { ChevronUp, ChevronDown, Loader2, Square, Trash2, Wallet, Clock, ArrowUpDown, ArrowUp, ArrowDown } from "lucide-react"; import { api, type ExecutorInfo, type ConsolidatedPosition } from "@/lib/api"; import { useServer } from "@/hooks/useServer"; +import { usePairBalances } from "@/hooks/usePairBalances"; import { useRates } from "@/hooks/useRates"; import { formatPct, formatAge, + formatPriceSig, formatUsd, pnlColor, isExecutorActive, @@ -35,13 +37,6 @@ interface TradeBottomPaneProps { const STORAGE_KEY = "condor_trade_bottom_pane"; -function formatPrice(price: number): string { - if (price === 0) return "—"; - if (Math.abs(price) >= 1000) return price.toFixed(2); - if (Math.abs(price) >= 1) return price.toFixed(4); - return price.toPrecision(6); -} - const STRATEGY_LABELS: Record = { LIMIT: "Limit", LIMIT_CHASER: "Chaser", @@ -136,14 +131,14 @@ function ExecutorTooltip({
{entry > 0 && ( - Entry: {formatPrice(entry)} + Entry: {formatPriceSig(entry)} )} {entry > 0 && exit > 0 && exit !== entry && →} {exit > 0 && exit !== entry && ( {active ? "Now" : "Close"}:{" "} - {formatPrice(exit)} + {formatPriceSig(exit)} )}
@@ -179,7 +174,7 @@ function ExecutorTooltip({
Range: - {formatPrice(Number(config.start_price as number))} – {formatPrice(Number(config.end_price as number))} + {formatPriceSig(Number(config.start_price as number))} – {formatPriceSig(Number(config.end_price as number))}
)} @@ -266,31 +261,14 @@ export function TradeBottomPane({ try { localStorage.setItem(STORAGE_KEY, expanded ? "1" : "0"); } catch { /* ok */ } }, [expanded]); - // Fetch balances for the active connector - const { data: portfolio } = useQuery({ - queryKey: ["portfolio", server], - queryFn: () => api.getPortfolio(server!), - enabled: !!server, - refetchInterval: 30_000, - }); - // Extract base/quote tokens from pair (e.g. "BTC-USDT" -> ["BTC", "USDT"]), // unless the caller knows better — see baseSymbol/quoteSymbol. const [pairBase, pairQuote] = pair.split("-"); const baseToken = baseSymbol || pairBase; const quoteToken = quoteSymbol || pairQuote; - // Find balances for the active connector - const connectorBalances = portfolio?.connectors?.find( - (c) => c.connector === connector, - )?.balances; - - const baseBalance = connectorBalances?.find( - (b) => b.token.toUpperCase() === (baseToken ?? "").toUpperCase(), - ); - const quoteBalance = connectorBalances?.find( - (b) => b.token.toUpperCase() === (quoteToken ?? "").toUpperCase(), - ); + // Balances for the active connector (shared ["portfolio", server] query) + const balances = usePairBalances(server, connector, baseToken, quoteToken); const stopMutation = useMutation({ mutationFn: (id: string) => { @@ -304,6 +282,9 @@ export function TradeBottomPane({ return next; }); setConfirmStopId(null); + // Prefix match covers ["executors", server, "main", pair] and the altPair variant + queryClient.invalidateQueries({ queryKey: ["executors", server] }); + queryClient.invalidateQueries({ queryKey: ["dex-lp-executors", server] }); queryClient.invalidateQueries({ queryKey: ["executors-infinite", server] }); }, }); @@ -327,7 +308,6 @@ export function TradeBottomPane({ return next; }); setConfirmClearPos(null); - queryClient.invalidateQueries({ queryKey: ["positions", server] }); queryClient.invalidateQueries({ queryKey: ["consolidated-positions", server] }); }, }); @@ -436,17 +416,17 @@ export function TradeBottomPane({ {/* Balance indicators */}
- {isSpot && baseBalance && ( + {isSpot && balances.base !== null && ( - {formatBalance(baseBalance.available)} {baseToken} + {formatBalance(balances.base)} {baseToken} )} - {quoteBalance && ( + {balances.quote !== null && ( - {formatBalance(quoteBalance.available)} {quoteToken} + {formatBalance(balances.quote)} {quoteToken} )} - {!baseBalance && !quoteBalance && ( + {balances.base === null && balances.quote === null && ( — )}
@@ -567,10 +547,10 @@ export function TradeBottomPane({ )} - {entry > 0 ? formatPrice(entry) : "—"} + {entry > 0 ? formatPriceSig(entry) : "—"} - {exit > 0 && exit !== entry ? formatPrice(exit) : "—"} + {exit > 0 && exit !== entry ? formatPriceSig(exit) : "—"} {pos.position_side?.toUpperCase() ?? "—"} + {pos.pool_scoped === false && ( + + any pool + + )}
{Math.abs(pos.amount).toFixed(4)} (${(pos.notional_value ?? Math.abs(pos.amount) * pos.entry_price).toFixed(2)}) - @ {formatPrice(pos.entry_price)} + @ {formatPriceSig(pos.entry_price)}
{!isFlat && ( - Now: {formatPrice(pos.current_price)} + Now: {formatPriceSig(pos.current_price)} )} {pos.leverage > 1 && ( diff --git a/frontend/src/components/trade/TradeChart.tsx b/frontend/src/components/trade/TradeChart.tsx index 77fed9116..8bac28847 100644 --- a/frontend/src/components/trade/TradeChart.tsx +++ b/frontend/src/components/trade/TradeChart.tsx @@ -3,11 +3,11 @@ import { createPortal } from "react-dom"; import { useCandleStore } from "@/hooks/useCandleStore"; import { api, type ConsolidatedPosition } from "@/lib/api"; -import { candleStore } from "@/lib/candle-store"; +import { candleChannelKey, candleStore } from "@/lib/candle-store"; import type { ExtraLine, PickSlot } from "@/components/executor/types"; import { getExecutorColor, type ExecutorOverlay } from "@/lib/executor-overlays"; import { getThemeColors, pnlHexColor, sideColor } from "@/lib/theme-colors"; -import { escapeHtml, formatCompactUsd } from "@/lib/formatters"; +import { escapeHtml, formatCompactUsd, formatPriceSig } from "@/lib/formatters"; type PickField = PickSlot | null; @@ -56,6 +56,8 @@ interface TradeChartProps { totalAmountQuote?: number; minOrderAmountQuote?: number; activePickField: PickField; + /** Per-slot overrides for the line titles and pick hints (see ChartPriceMapping). */ + lineLabels?: Partial>; onPriceSet: (field: PickSlot, price: number) => void; pricePrecision?: number; extraLines?: ExtraLine[]; @@ -65,8 +67,6 @@ interface TradeChartProps { /** Convert a value from the pair's quote currency to display currency */ convertValue?: (val: number) => string; convertPnl?: (val: number) => string; - /** Callback when user clicks an executor in the list that's outside candle range */ - onRequestCandleRange?: (startTime: number) => void; /** Callback when user clicks chart background to deselect executor */ onExecutorDeselect?: () => void; /** Called once the series exists, with the chart's price mapping. */ @@ -88,6 +88,7 @@ export function TradeChart({ totalAmountQuote, minOrderAmountQuote, activePickField, + lineLabels, onPriceSet, pricePrecision, extraLines, @@ -142,18 +143,20 @@ export function TradeChart({ ); // ── Filter executor overlays to those within candle time range ── + // Depend on the earliest candle timestamp (not the candles array, whose reference + // changes on every WS tick) so filteredOverlays keeps a stable identity across + // live ticks and the overlay rebuild effect below doesn't churn per tick. + const minCandleTime = candles.length ? candles[0].timestamp : 0; const filteredOverlays = useMemo(() => { if (!executorOverlays?.length) return executorOverlays; - if (!candles.length) return executorOverlays; // no candles yet → show all - const minTime = candles[0].timestamp; return executorOverlays.filter((o) => { const s = o.status?.toLowerCase(); if (s === "running" || s === "active") return true; if (selectedExecutorId && o.executorId === selectedExecutorId) return true; const end = o.timeRange.end > 1e12 ? o.timeRange.end / 1000 : o.timeRange.end; - return end >= minTime; + return end >= minCandleTime; // minCandleTime === 0 (no candles) → show all }); - }, [executorOverlays, candles, selectedExecutorId]); + }, [executorOverlays, minCandleTime, selectedExecutorId]); // ── REST backfill on pair/interval/lookback change ── const backfillKeyRef = useRef(""); @@ -389,12 +392,6 @@ export function TradeChart({ })(); let detailRows = ""; - const fmtPrice = (p: number) => { - if (p === 0) return "—"; - if (Math.abs(p) >= 1000) return p.toFixed(2); - if (Math.abs(p) >= 1) return p.toFixed(4); - return p.toPrecision(6); - }; const addRow = (label: string, value: string, color?: string) => { detailRows += `
${escapeHtml(label)}${escapeHtml(value)}
`; }; @@ -404,18 +401,18 @@ export function TradeChart({ if (o.gridBox) { if (o.type === "lp") { // startPrice is the box's upper edge (see computeLpOverlay). - addRow("Upper Price", fmtPrice(o.gridBox.startPrice)); - addRow("Lower Price", fmtPrice(o.gridBox.endPrice)); + addRow("Upper Price", formatPriceSig(o.gridBox.startPrice)); + addRow("Lower Price", formatPriceSig(o.gridBox.endPrice)); if (cfg.lp_provider != null) addRow("Provider", String(cfg.lp_provider)); } else { - addRow("Start Price", fmtPrice(o.gridBox.startPrice)); - addRow("End Price", fmtPrice(o.gridBox.endPrice)); - if (o.gridBox.limitPrice) addRow("Limit Price", fmtPrice(o.gridBox.limitPrice)); + addRow("Start Price", formatPriceSig(o.gridBox.startPrice)); + addRow("End Price", formatPriceSig(o.gridBox.endPrice)); + if (o.gridBox.limitPrice) addRow("Limit Price", formatPriceSig(o.gridBox.limitPrice)); } } else if (o.entryPrice && o.entryPrice > 0) { - addRow("Entry", fmtPrice(o.entryPrice)); + addRow("Entry", formatPriceSig(o.entryPrice)); if (o.exitPrice && o.exitPrice > 0 && o.exitPrice !== o.entryPrice) { - addRow(o.status?.toLowerCase() === "running" ? "Current" : "Close", fmtPrice(o.exitPrice)); + addRow(o.status?.toLowerCase() === "running" ? "Current" : "Close", formatPriceSig(o.exitPrice)); } } @@ -547,7 +544,7 @@ export function TradeChart({ useEffect(() => { if (!chartReady || !seriesRef.current || !candles.length) return; - const key = `${server}:${connector}:${pair}:${interval}`; + const key = candleChannelKey(server, connector, pair, interval, poolAddress); const first = candles[0].timestamp; const prevSig = lastSetDataSigRef.current; const [prevKey, prevFirstStr, prevLenStr] = prevSig.split("|"); @@ -586,12 +583,12 @@ export function TradeChart({ chartRef.current?.timeScale().fitContent(); initializedRef.current = true; } - }, [candles, chartReady, server, connector, pair, interval]); + }, [candles, chartReady, server, connector, pair, interval, poolAddress]); // ── Real-time last candle update via candle store listener ── useEffect(() => { if (!chartReady || !seriesRef.current) return; - const key = `candles:${server}:${connector}:${pair}:${interval}`; + const key = candleChannelKey(server, connector, pair, interval, poolAddress); const removeListener = candleStore.onUpdate(key, (updated) => { if (!seriesRef.current || !updated.length) return; @@ -606,7 +603,7 @@ export function TradeChart({ }); return removeListener; - }, [chartReady, server, connector, pair, interval]); + }, [chartReady, server, connector, pair, interval, poolAddress]); // ── Reset auto-fit on pair/interval/range change ── useEffect(() => { @@ -619,7 +616,7 @@ export function TradeChart({ seriesRef.current.applyOptions({ priceFormat: { type: "price" as const, precision: pricePrecision, minMove: 1 / 10 ** pricePrecision }, }); - }, [pricePrecision]); + }, [pricePrecision, chartReady]); // ── Price lines (start/end/limit/grid levels/extras) ── useEffect(() => { @@ -643,7 +640,7 @@ export function TradeChart({ lineWidth: 2, lineStyle: mod.LineStyle.Solid, axisLabelVisible: true, - title: "Start", + title: lineLabels?.start ?? "Start", }); } @@ -654,7 +651,7 @@ export function TradeChart({ lineWidth: 2, lineStyle: mod.LineStyle.Dashed, axisLabelVisible: true, - title: "End", + title: lineLabels?.end ?? "End", }); } @@ -666,7 +663,7 @@ export function TradeChart({ lineWidth: 2, lineStyle: mod.LineStyle.Dotted, axisLabelVisible: true, - title: "Limit", + title: lineLabels?.limit ?? "Limit", }); } @@ -724,7 +721,7 @@ export function TradeChart({ extraLinesRef.current.push(pl); } } - }, [startPrice, endPrice, limitPrice, side, minSpread, totalAmountQuote, minOrderAmountQuote, activePickField, extraLines, chartReady]); + }, [startPrice, endPrice, limitPrice, side, minSpread, totalAmountQuote, minOrderAmountQuote, activePickField, extraLines, lineLabels, chartReady]); // ── Executor overlays ── // `chartReady` is a dependency, not a guard for its own sake: lightweight-charts @@ -952,7 +949,11 @@ export function TradeChart({ const pnlStr = _cvtPnl2 ? _cvtPnl2(pnl) : (Math.abs(pnl) >= 1000 ? `${pnl >= 0 ? "+" : ""}$${(pnl / 1000).toFixed(1)}K` : `${pnl >= 0 ? "+" : ""}$${pnl.toFixed(2)}`); const amt = Math.abs(pos.amount); const color = pnlHexColor(pnl); - const label = `${isLong ? "LONG" : "SHORT"} ${amt.toFixed(4)} · ${pnlStr}`; + // A hold nothing ties to this pool still belongs on the chart -- the price + // is the same market -- but it must not read as this pool's position, so + // it says whose it is: the pair's, across whatever pool the router used. + const scope = pos.pool_scoped === false ? " · any pool" : ""; + const label = `${isLong ? "LONG" : "SHORT"} ${amt.toFixed(4)} · ${pnlStr}${scope}`; const pl = series.createPriceLine({ price: pos.entry_price, color, @@ -1009,10 +1010,13 @@ export function TradeChart({ {activePickField && (

- Click on chart to set {PICK_LABELS[activePickField]} price + Click on chart to set{" "} + {(lineLabels?.[activePickField] ?? PICK_LABELS[activePickField]).toLowerCase()}{" "} + price

- Pick mode: {PICK_LABELS[activePickField]} + Pick mode:{" "} + {(lineLabels?.[activePickField] ?? PICK_LABELS[activePickField]).toLowerCase()}
)} diff --git a/frontend/src/components/ui/AnchoredMenu.tsx b/frontend/src/components/ui/AnchoredMenu.tsx new file mode 100644 index 000000000..aafd5d373 --- /dev/null +++ b/frontend/src/components/ui/AnchoredMenu.tsx @@ -0,0 +1,108 @@ +import { useEffect, useLayoutEffect, useRef, useState } from "react"; +import { createPortal } from "react-dom"; + +interface Props { + /** The button the panel hangs off. */ + anchor: HTMLElement | null; + open: boolean; + onClose: () => void; + /** Which edge of the panel lines up with the same edge of the anchor. */ + align?: "left" | "right"; + /** Width utility for the panel, e.g. `w-44`. */ + className?: string; + children: React.ReactNode; +} + +/** + * A dropdown panel that no ancestor can clip. + * + * An `absolute` menu is still a child of its trigger, so any `overflow-hidden` + * on the way up cuts it off. That is exactly what happened to the DEX chain and + * venue pickers: the browser wraps its header and pool table in one clipping + * card, and on a chain with no pools that card is barely taller than the header + * itself — so most of the chain list was invisible and unclickable, on the very + * screen where switching chains is the only thing left to do. + * + * Rendering into `document.body` at fixed coordinates escapes every clipping + * ancestor. The trade is that the panel no longer moves with the page, so it is + * re-anchored on scroll and resize instead. + */ +export function AnchoredMenu({ + anchor, + open, + onClose, + align = "left", + className = "", + children, +}: Props) { + const menuRef = useRef(null); + const [pos, setPos] = useState<{ + top: number; + left?: number; + right?: number; + } | null>(null); + + // Stale coordinates from a previous open are harmless: the panel renders + // nothing while closed, and this runs before the browser paints the reopen. + useLayoutEffect(() => { + if (!open || !anchor) return; + const place = () => { + const r = anchor.getBoundingClientRect(); + setPos( + align === "right" + ? { top: r.bottom + 6, right: Math.max(8, window.innerWidth - r.right) } + : { top: r.bottom + 6, left: Math.max(8, r.left) }, + ); + }; + place(); + // Capture phase: the page's own scroll containers do not bubble scroll. + window.addEventListener("scroll", place, true); + window.addEventListener("resize", place); + return () => { + window.removeEventListener("scroll", place, true); + window.removeEventListener("resize", place); + }; + }, [open, anchor, align]); + + useEffect(() => { + if (!open) return; + // The panel is no longer inside the trigger's subtree, so "outside" has to be + // measured against both. Miss the panel and a mousedown on a menu item would + // unmount it before the click that selects it ever lands. + const onDown = (e: MouseEvent) => { + const target = e.target as Node; + if (menuRef.current?.contains(target) || anchor?.contains(target)) return; + onClose(); + }; + const onKey = (e: KeyboardEvent) => { + if (e.key === "Escape") onClose(); + }; + document.addEventListener("mousedown", onDown); + document.addEventListener("keydown", onKey); + return () => { + document.removeEventListener("mousedown", onDown); + document.removeEventListener("keydown", onKey); + }; + }, [open, anchor, onClose]); + + if (!open || !pos) return null; + + return createPortal( +
+ {children} +
, + document.body, + ); +} diff --git a/frontend/src/hooks/useCandleStore.ts b/frontend/src/hooks/useCandleStore.ts index 1a011f775..2dffc18ea 100644 --- a/frontend/src/hooks/useCandleStore.ts +++ b/frontend/src/hooks/useCandleStore.ts @@ -1,7 +1,7 @@ import { useEffect, useRef, useState } from "react"; import type { CandleData } from "@/lib/api"; -import { candleStore } from "@/lib/candle-store"; +import { candleChannelKey, candleStore } from "@/lib/candle-store"; /** Staleness thresholds by interval category */ const STALE_THRESHOLD_SUB_1H_MS = 30_000; // 30s for intervals < 1h @@ -38,10 +38,7 @@ export function useCandleStore( mergeCandles: (c: CandleData[]) => void; setDuration: (seconds: number) => void; } { - const key = server - ? `candles:${server}:${connector}:${pair}:${interval}` + - (poolAddress ? `:${poolAddress}` : "") - : ""; + const key = server ? candleChannelKey(server, connector, pair, interval, poolAddress) : ""; const [candles, setCandles] = useState([]); const [isStale, setIsStale] = useState(false); diff --git a/frontend/src/hooks/useChatSocket.ts b/frontend/src/hooks/useChatSocket.ts index 4dcf83d45..59231fa59 100644 --- a/frontend/src/hooks/useChatSocket.ts +++ b/frontend/src/hooks/useChatSocket.ts @@ -1,5 +1,11 @@ import { useCallback, useEffect, useRef, useState } from "react"; -import { api, type ConversationTurn } from "@/lib/api"; +import { useQueryClient } from "@tanstack/react-query"; +import { + api, + type AppNotification, + type ConversationTurn, + type NotificationsResponse, +} from "@/lib/api"; import { useAuth } from "@/lib/auth"; import { toolCallState } from "@/lib/formatters"; import { getViewContext } from "@/lib/viewContext"; @@ -224,8 +230,19 @@ function turnsToMessages(turns: ConversationTurn[]): ChatMessage[] { */ const FLUSH_INTERVAL_MS = 50; +/** + * The bell's cache key (FEAT-048). + * + * Lives here because this is where the live `notification` event is written + * into it; `NotificationBell` reads the same key, so a pushed notice and a + * fetched one are one list. Not server-scoped: notifications belong to the + * user, not to whichever trading server is selected. + */ +export const NOTIFICATIONS_KEY = ["notifications"] as const; + export function useChatSocket() { const { token, user } = useAuth(); + const queryClient = useQueryClient(); const wsRef = useRef(null); const reconnectTimer = useRef>(undefined); // Whether this hook still wants a socket. `close()` is asynchronous, so the @@ -959,6 +976,37 @@ export function useChatSocket() { break; } + case "notification": { + // A background task finished (FEAT-048). Addressed to the *user*, not + // to a conversation, so it carries no slot and goes nowhere near the + // transcript — it lands in the bell's react-query cache, which is the + // same place `GET /notifications` fills on mount. Writing into the + // cache rather than into local state is what lets the bell be a leaf + // component with no wiring back up to here. + const incoming: AppNotification = { + id: (data.id as string) || "", + user_id: 0, + ts: (data.ts as number) || Date.now() / 1000, + kind: (data.kind as string) || "system", + text: (data.text as string) || "", + title: (data.title as string) || null, + link: (data.link as string) || null, + read: false, + }; + if (!incoming.id) break; + queryClient.setQueryData( + NOTIFICATIONS_KEY, + (prev) => { + // A reconnect can replay one we already hold; keyed by id so it + // is never listed twice. + const rest = (prev?.items ?? []).filter((n) => n.id !== incoming.id); + const items = [incoming, ...rest].slice(0, 50); + return { items, unread: items.filter((n) => !n.read).length }; + }, + ); + break; + } + case "heartbeat": break; } @@ -969,6 +1017,7 @@ export function useChatSocket() { flushChunks, hydrateSlot, prewarmLatest, + queryClient, send, stopStreaming, ], diff --git a/frontend/src/hooks/useMainControllerData.ts b/frontend/src/hooks/useMainControllerData.ts index 3d38fc223..fee0375a6 100644 --- a/frontend/src/hooks/useMainControllerData.ts +++ b/frontend/src/hooks/useMainControllerData.ts @@ -106,21 +106,27 @@ export function useMainControllerData( // compared six fields, which made it both redundant here and a staleness // trap -- a change to any seventh field kept serving the old rows. const executors = useMemo(() => { - const seen = new Set(); - const rows: ExecutorInfo[] = []; // The by-pool rows are the whole server's LP history, so they are admitted // only on an exact pool match -- unlike the pair rows, which were already // scoped by the query that fetched them. const inThisPool = poolAddress ? (poolLpExecutors ?? []).filter((ex) => executorPool(ex) === poolAddress) : []; + // Upsert by id, by-pool rows last: they refresh on their own 20s poll, + // while the pair queries sit still between WS pushes -- so when both hold + // the same executor, the by-pool copy replaces the pair copy instead of + // being shadowed by it. Map keeps first-insertion order, so row order is + // unchanged from the dedup this replaces. + const byId = new Map(); for (const ex of [ ...(cachedExecutors ?? []), ...(wantsAlt ? (altExecutors ?? []) : []), ...inThisPool, ]) { - if (seen.has(ex.id)) continue; - seen.add(ex.id); + byId.set(ex.id, ex); + } + const rows: ExecutorInfo[] = []; + for (const ex of byId.values()) { if (ex.connector !== connector) continue; if (poolAddress && ex.type?.toLowerCase() === "lp") { const pool = executorPool(ex); @@ -143,6 +149,26 @@ export function useMainControllerData( placeholderData: keepPreviousData, // keep showing old data during refetch/refresh }); + /** + * Executor id -> the pool it traded in, for every row we hold. + * + * Built from the *unfiltered* queries on purpose: the pool filter above drops + * a foreign pool's LP executor, and that row is exactly the evidence needed to + * recognise a held position as belonging somewhere else. + */ + const executorPools = useMemo(() => { + const byId = new Map(); + for (const ex of [ + ...(cachedExecutors ?? []), + ...(altExecutors ?? []), + ...(poolLpExecutors ?? []), + ]) { + const pool = executorPool(ex); + if (pool) byId.set(ex.id, pool); + } + return byId; + }, [cachedExecutors, altExecutors, poolLpExecutors]); + const positions = useMemo(() => { if (!positionsData) return []; const all = [ @@ -157,14 +183,41 @@ export function useMainControllerData( ...(wantsAlt ? [altPair] : []), ...executors.map((ex) => ex.trading_pair).filter(Boolean), ]); - return all.filter( - (p) => - // Show positions from main controller or untagged (executor-level positions) - (!p.controller_id || p.controller_id === "main") && - p.connector_name === connector && - pairs.has(p.trading_pair), - ); - }, [positionsData, connector, pair, altPair, wantsAlt, executors]); + return all + .filter( + (p) => + // Show positions from main controller or untagged (executor-level positions) + (!p.controller_id || p.controller_id === "main") && + p.connector_name === connector && + pairs.has(p.trading_pair), + ) + .map((p) => { + // On a DEX pool page the connector is the *network*, so connector+pair + // alone puts one SOL-USDC hold on every Solana SOL-USDC pool -- Orca's + // and Meteora's alike -- and each page reads as if the position were + // its own. The executors behind the row are the only attribution there + // is: an LP executor names its pool, and a swap names none because the + // aggregator picked the route. + if (!poolAddress) return p; + const pools = (p.executor_ids ?? []) + .map((id) => executorPools.get(id)) + .filter((pool): pool is string => !!pool); + if (!pools.length) return { ...p, pool_scoped: false }; + return pools.includes(poolAddress) + ? { ...p, pool_scoped: true } + : null; + }) + .filter((p): p is NonNullable => p !== null); + }, [ + positionsData, + connector, + pair, + altPair, + wantsAlt, + executors, + poolAddress, + executorPools, + ]); return { executors, overlays, positions, isLoadingPositions }; } diff --git a/frontend/src/hooks/useTelemetry.ts b/frontend/src/hooks/useTelemetry.ts new file mode 100644 index 000000000..2405ddec5 --- /dev/null +++ b/frontend/src/hooks/useTelemetry.ts @@ -0,0 +1,46 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { api } from "@/lib/api"; +import type { TelemetryLevel, TelemetrySettingsResponse } from "@/lib/api"; + +export const TELEMETRY_KEY = ["telemetry-settings"] as const; + +/** + * The install's telemetry state, shared by the two places that render it: the + * consent banner in the shell and the card in Settings → Privacy. One query key, + * so opening Settings right after answering the banner costs no second request + * and cannot show a staler answer than the one just given. + */ +export function useTelemetry() { + return useQuery({ + queryKey: TELEMETRY_KEY, + queryFn: api.getTelemetrySettings, + // Nothing else in the app changes this, and it survives restarts on disk. + staleTime: 5 * 60 * 1000, + retry: false, + }); +} + +export function useSetTelemetryLevel() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (level: TelemetryLevel) => api.setTelemetryLevel(level), + onSuccess: (res) => { + // Write the answer through rather than refetching: the banner unmounts on + // `consent !== "unknown"`, and a refetch round-trip would leave it on + // screen long enough to look like the click did nothing. + qc.setQueryData(TELEMETRY_KEY, (old: TelemetrySettingsResponse | undefined) => + old ? { ...old, level: res.level, consent: "granted" as const } : old, + ); + }, + }); +} + +/** + * Should we ask? Only an install that has never answered, only the admin who is + * allowed to answer, and never when the environment has pinned the level — in + * that case there is no answer left to give. + */ +export function shouldAskConsent(data: TelemetrySettingsResponse | undefined): boolean { + return !!data && data.consent === "unknown" && data.can_change && !data.env_overridden; +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index dae2d617c..dddca9c41 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -14,6 +14,25 @@ async function apiFetch(path: string, init?: RequestInit): Promise { return res.json(); } +/** A finished background task, addressed to the user rather than to a chat. */ +export interface AppNotification { + id: string; + user_id: number; + ts: number; + /** "delegation" | "routine" | "agent" | "system" */ + kind: string; + text: string; + title?: string | null; + /** A dashboard route to open, e.g. "/routines?tab=reports". */ + link?: string | null; + read: boolean; +} + +export interface NotificationsResponse { + items: AppNotification[]; + unread: number; +} + // ── Types ── export interface ServerInfo { @@ -22,6 +41,8 @@ export interface ServerInfo { port: number; online: boolean; permission: string; + /** The server commands fall back to when none is named — shared with Telegram. */ + is_default: boolean; } export interface BalanceItem { @@ -230,10 +251,21 @@ export interface ConsolidatedPosition { realized_pnl: number; cum_fees: number; executor_count: number; + /** The executors this row aggregates — how a hold is traced back to a pool. */ + executor_ids?: string[]; leverage: number; controller_id: string; source: "executor" | "bot"; source_name: string; + /** + * Set only on a pool workspace: whether this hold is known to be *this* pool's. + * + * `true` when a contributing LP executor records the pool on screen, `false` + * when nothing ties it to any pool — a swap is routed by the aggregator, so + * the position it leaves behind is a fact about the pair, not about a pool. + * Undefined off a pool page, where the question does not arise. + */ + pool_scoped?: boolean; } export interface ConsolidatedPositionsResponse { @@ -387,6 +419,13 @@ export interface DexUpstream { */ export type DexTokenVerdict = "listed" | "added" | "symbol_taken" | "failed"; +/** The token that already holds a ticker a new token wanted. */ +export interface DexTokenConflict { + symbol: string; + address: string; + name?: string | null; +} + /** A page of pools. `has_more`, not a count — no upstream reports a total. */ export interface PoolPage { pools: PoolSummary[]; @@ -918,6 +957,8 @@ export interface GatewayNetworkInfo { export interface GatewayNetworkConfig { network_id: string; config: Record; + /** True when the server masked credential-bearing URL values (non-owner read, SEC-197). */ + redacted?: boolean; } export interface GatewayWalletGroup { @@ -959,6 +1000,43 @@ export interface VoiceSettingsResponse { available_languages: Record; } +// ── Telemetry (FEAT-023) ── + +/** The only two answers. `off` is reachable only from the environment. */ +export type TelemetryLevel = "ping" | "usage"; + +export interface TelemetryOption { + level: TelemetryLevel; + label: string; +} + +/** + * What the install is told before it answers, served by the backend so this + * copy and the Telegram prompt's cannot drift (`condor/telemetry/prompt.py`). + */ +export interface TelemetryDisclosure { + headline: string; + always_on: string; + optional: string; + never: string[]; + doc: string; + options: TelemetryOption[]; +} + +export interface TelemetrySettingsResponse { + /** `unknown` until someone answers — that is what the consent card asks. */ + consent: "unknown" | "granted" | "denied"; + level: "off" | TelemetryLevel; + /** `CONDOR_TELEMETRY` is pinned in the environment; nothing here can change it. */ + env_overridden: boolean; + endpoint_configured: boolean; + pending_events: number; + privacy_doc: string; + /** Consent is install-wide, so only the admin may answer. */ + can_change: boolean; + disclosure: TelemetryDisclosure; +} + // ── Chat ── export interface ChatAgentOption { @@ -1470,6 +1548,27 @@ export const api = { { method: "POST", body: JSON.stringify({ network, addresses }) }, ), + /** + * Register one token because the user asked to, where the automatic ensure + * only reported. Retries a `failed` verdict outright; on `symbol_taken` the + * answer names the ticker's current holder, and a second call with + * `replace: true` deletes that holder and registers this token in its place. + */ + addDexToken: ( + server: string, + network: string, + address: string, + symbol?: string, + replace?: boolean, + ) => + apiFetch<{ verdict: DexTokenVerdict; conflict: DexTokenConflict | null }>( + `/api/v1/servers/${encodeURIComponent(server)}/dex/tokens/add`, + { + method: "POST", + body: JSON.stringify({ network, address, symbol, replace }), + }, + ), + /** One pool by address, so /dex/{network}/{address} renders from the URL alone. */ getDexPoolByAddress: (server: string, address: string, network: string) => apiFetch( @@ -2080,6 +2179,18 @@ export const api = { body: JSON.stringify(data), }), + // ── Telemetry (FEAT-023) ── + + getTelemetrySettings: () => + apiFetch("/api/v1/settings/telemetry"), + + /** Admin only (403 otherwise), and 409 when `CONDOR_TELEMETRY` is pinned. */ + setTelemetryLevel: (level: TelemetryLevel) => + apiFetch<{ level: TelemetryLevel; consent: string }>( + `/api/v1/settings/telemetry?level=${level}`, + { method: "PUT" }, + ), + // ── Chat ── getOpenRouterModels: () => @@ -2170,4 +2281,17 @@ export const api = { `/api/v1/settings/custom-providers/${encodeURIComponent(name)}`, { method: "DELETE" }, ), + + // ── Notifications (FEAT-048) ── + + /** The bell's history. Scoped to the JWT server-side — there is no user param. */ + getNotifications: (limit = 50) => + apiFetch(`/api/v1/notifications?limit=${limit}`), + + /** Mark some notifications read, or all of them when `ids` is omitted. */ + markNotificationsRead: (ids?: string[]) => + apiFetch<{ unread: number }>("/api/v1/notifications/read", { + method: "POST", + body: JSON.stringify({ ids: ids ?? null }), + }), }; diff --git a/frontend/src/lib/auth.ts b/frontend/src/lib/auth.ts index 81dc17670..476024770 100644 --- a/frontend/src/lib/auth.ts +++ b/frontend/src/lib/auth.ts @@ -21,6 +21,8 @@ export interface AuthState { token: string | null; isAuthenticated: boolean; loginWithToken: (loginToken: string) => Promise; + /** Local mode (FEAT-049): claim the local admin's session. 404s elsewhere. */ + loginLocal: () => Promise; logout: () => void; } @@ -34,6 +36,7 @@ export const AuthContext = createContext({ token: null, isAuthenticated: false, loginWithToken: async () => {}, + loginLocal: async () => {}, logout: () => {}, }); @@ -67,6 +70,23 @@ export function useAuthState(): AuthState { setUser(data.user); }, []); + // Local mode: the dashboard runs on this machine with no login at all, so + // the session is claimed rather than presented. Stores exactly what + // loginWithToken stores — downstream there is no such thing as a "local" + // session, only a session. + const loginLocal = useCallback(async () => { + const res = await fetch("/api/v1/auth/local-login", { method: "POST" }); + if (!res.ok) { + const err = await res.json().catch(() => ({})); + throw new Error(err.detail || "Local login failed"); + } + const data = await res.json(); + localStorage.setItem(TOKEN_KEY, data.token); + localStorage.setItem(USER_KEY, JSON.stringify(data.user)); + setToken(data.token); + setUser(data.user); + }, []); + const logout = useCallback(() => { localStorage.removeItem(TOKEN_KEY); localStorage.removeItem(USER_KEY); @@ -99,6 +119,7 @@ export function useAuthState(): AuthState { token, isAuthenticated: !!token && !!user, loginWithToken, + loginLocal, logout, }; } diff --git a/frontend/src/lib/candle-store.ts b/frontend/src/lib/candle-store.ts index f30dcec8a..2d71dd437 100644 --- a/frontend/src/lib/candle-store.ts +++ b/frontend/src/lib/candle-store.ts @@ -41,6 +41,26 @@ function normalizeTimestamp(ts: number): number { return ts > 1e12 ? ts / 1000 : ts; } +/** + * Canonical candle store/WS channel key. The optional pool segment pins the + * stream to one DEX pool; omitted (every CLOB chart) the key keeps its five + * segments, byte for byte. Every consumer must build keys through this — + * hand-rebuilt templates drift (a missing pool segment subscribes a listener + * to a channel that never notifies). + */ +export function candleChannelKey( + server: string, + connector: string, + pair: string, + interval: string, + poolAddress?: string, +): string { + return ( + `candles:${server}:${connector}:${pair}:${interval}` + + (poolAddress ? `:${poolAddress}` : "") + ); +} + function sortedFromMap(map: Map): CandleData[] { return Array.from(map.values()).sort((a, b) => a.timestamp - b.timestamp); } diff --git a/frontend/src/lib/clipboard.ts b/frontend/src/lib/clipboard.ts new file mode 100644 index 000000000..23f8731db --- /dev/null +++ b/frontend/src/lib/clipboard.ts @@ -0,0 +1,34 @@ +/** + * Copy text to the clipboard, working in both secure and insecure contexts. + * + * `navigator.clipboard` only exists in secure contexts (https/localhost); the + * self-hosted dashboard is often served over plain http on a LAN/VPS IP, where + * it is undefined. In that case fall back to a hidden textarea + + * `document.execCommand("copy")`. + * + * Resolves `true` when the text was copied, `false` on failure — never throws. + */ +export async function copyText(text: string): Promise { + if (navigator.clipboard?.writeText) { + try { + await navigator.clipboard.writeText(text); + return true; + } catch { + // Permission denied or transient failure — try the legacy path below. + } + } + try { + const textarea = document.createElement("textarea"); + textarea.value = text; + textarea.setAttribute("readonly", ""); + textarea.style.position = "fixed"; + textarea.style.opacity = "0"; + document.body.appendChild(textarea); + textarea.select(); + const ok = document.execCommand("copy"); + document.body.removeChild(textarea); + return ok; + } catch { + return false; + } +} diff --git a/frontend/src/lib/formatters.ts b/frontend/src/lib/formatters.ts index 4806606f6..eddc37e6c 100644 --- a/frontend/src/lib/formatters.ts +++ b/frontend/src/lib/formatters.ts @@ -163,6 +163,22 @@ export function formatRelativeTime( return `${Math.floor(diff / 86400)}d ago`; } +/** + * Significant-digits price formatter shared by the trade pane, the chart + * tooltip, and the LP position bar — the same entry price must render as the + * identical string in all three. Prices span memecoin to majors, hence + * significant digits below 1 rather than fixed decimals. Em-dash for a + * missing/zero price (an executor with no entry yet, an open LP bound). + * Distinct from `formatPrice` below, which locale-groups and uses 4 + * significant digits — its call sites keep their rendering. + */ +export function formatPriceSig(val: number | null | undefined): string { + if (val == null || !Number.isFinite(val) || val === 0) return "—"; + if (Math.abs(val) >= 1000) return val.toFixed(2); + if (Math.abs(val) >= 1) return val.toFixed(4); + return val.toPrecision(6); +} + export function formatPrice(val: number): string { if (!val) return "\u2014"; if (val >= 1000) return val.toLocaleString("en-US", { maximumFractionDigits: 2 }); diff --git a/frontend/src/pages/CreateExecutor.tsx b/frontend/src/pages/CreateExecutor.tsx index ee9513550..09a0fe0c6 100644 --- a/frontend/src/pages/CreateExecutor.tsx +++ b/frontend/src/pages/CreateExecutor.tsx @@ -5,8 +5,6 @@ import { ArrowLeft, ArrowUpDown, BarChart3, - CheckCircle, - Copy, Droplets, Grid3X3, Layers, @@ -15,7 +13,6 @@ import { Rocket, Settings2, TrendingUp, - X, } from "lucide-react"; import { NoServerCard } from "@/components/NoServerCard"; @@ -26,6 +23,10 @@ import { MarketDepthPanel } from "@/components/market/MarketDepthPanel"; import { MarketsPanel } from "@/components/market/MarketsPanel"; import { TradeChart } from "@/components/trade/TradeChart"; import { GridConfigPanel, useGridValidation } from "@/components/grid/GridConfigPanel"; +import { + ErrorToast, + ExecutorSuccessModal, +} from "@/components/executor/ExecutorSuccessModal"; import { PositionConfigPanel, usePositionConfig } from "@/components/executor/PositionConfigPanel"; import { OrderConfigPanel, useOrderConfig } from "@/components/executor/OrderConfigPanel"; import { DCAConfigPanel, useDCAConfig } from "@/components/executor/DCAConfigPanel"; @@ -581,6 +582,7 @@ export function CreateExecutor() { side={chartProps.side} minSpread={chartProps.minSpread} activePickField={chartProps.activePickField} + lineLabels={chartProps.lineLabels} onPriceSet={handlePriceSet} pricePrecision={pricePrecision} extraLines={chartProps.extraLines} @@ -754,46 +756,12 @@ export function CreateExecutor() { {/* Success modal */} {successInfo && ( -
-
- {/* Close button */} - - - {/* Icon */} -
-
- -
-
- - {/* Title */} -

- {TYPE_LABELS[successInfo.type]} Created -

-

- Successfully deployed on {successInfo.connector} -

- - {/* Details */} -
-
- Executor ID -
- {successInfo.id.slice(0, 12)}… - -
-
+
Pair {successInfo.pair} @@ -805,24 +773,16 @@ export function CreateExecutor() { ACTIVE
-
- - {/* Action */} - -
-
+ + } + primaryLabel="Continue Trading" + onClose={() => setSuccessInfo(null)} + /> )} {/* Error toast */} {createMutation.isError && ( -
- {(createMutation.error as Error).message} -
+ )}
); diff --git a/frontend/src/pages/CreateGridExecutor.tsx b/frontend/src/pages/CreateGridExecutor.tsx index d007ff5b1..2bff9a459 100644 --- a/frontend/src/pages/CreateGridExecutor.tsx +++ b/frontend/src/pages/CreateGridExecutor.tsx @@ -1,9 +1,13 @@ import { useMutation, useQuery } from "@tanstack/react-query"; import { useEffect, useMemo, useReducer, useState } from "react"; import { useNavigate } from "react-router-dom"; -import { ArrowLeft, CheckCircle, Loader2, Rocket } from "lucide-react"; +import { ArrowLeft, Loader2, Rocket } from "lucide-react"; import { NoServerCard } from "@/components/NoServerCard"; +import { + ErrorToast, + ExecutorSuccessModal, +} from "@/components/executor/ExecutorSuccessModal"; import { ExchangeSelector } from "@/components/market/ExchangeSelector"; import { PairSelector, useTradingRules } from "@/components/market/PairSelector"; import { PriceTicker } from "@/components/market/PriceTicker"; @@ -314,29 +318,19 @@ export function CreateGridExecutor() { {/* Success modal */} {successId && ( -
-
- -
-

Grid Executor Created

-

{successId}

-
-

Redirecting to executors...

- -
-
+ navigate("/executors")} + onClose={() => setSuccessId(null)} + /> )} {/* Error toast */} {createMutation.isError && ( -
- {(createMutation.error as Error).message} -
+ )}
); diff --git a/frontend/src/pages/Dex.tsx b/frontend/src/pages/Dex.tsx index dc13e9764..689c8c044 100644 --- a/frontend/src/pages/Dex.tsx +++ b/frontend/src/pages/Dex.tsx @@ -1,4 +1,4 @@ -import { useQuery } from "@tanstack/react-query"; +import { useQuery, useQueryClient } from "@tanstack/react-query"; import { ArrowRight } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import { useNavigate } from "react-router-dom"; @@ -8,7 +8,7 @@ import { LpPositions } from "@/components/dex/LpPositions"; import { PoolBrowser } from "@/components/dex/PoolBrowser"; import { PoolSourceTabs } from "@/components/dex/PoolSourceTabs"; import { UpstreamNotice } from "@/components/dex/UpstreamNotice"; -import type { PoolSource } from "@/components/dex/pool-source"; +import { GATEWAY_TAB_CHAIN, type PoolSource } from "@/components/dex/pool-source"; import { useDexUpstream } from "@/hooks/useDexUpstream"; import { useServer } from "@/hooks/useServer"; import { api, type PoolSummary } from "@/lib/api"; @@ -19,6 +19,9 @@ const POOL_STALE_MS = 30_000; const SEARCH_DEBOUNCE_MS = 400; const PAGE_SIZE = 20; +/** GeckoTerminal's multi-pool endpoint takes at most 30 addresses per request. */ +const FAVORITES_BATCH_SIZE = 30; + /** * The chain the browser opens on. * @@ -47,6 +50,7 @@ const ADDRESS_RE = /^(0x[0-9a-fA-F]{40}|[1-9A-HJ-NP-Za-km-z]{32,44})$/; export function Dex() { const { server } = useServer(); const navigate = useNavigate(); + const queryClient = useQueryClient(); const { favorites } = useDexFavorites(); // Everything on this page comes from one shared GeckoTerminal budget; when it // is spent the tables empty out, so the reason is shown above them. @@ -100,7 +104,7 @@ export function Dex() { setDexes([]); // Meteora and Orca are Solana connectors; their tabs disappear off Solana, and // a selection left pointing at a hidden tab shows a table with no active tab. - if (!next.startsWith("solana")) { + if (!next.startsWith(GATEWAY_TAB_CHAIN)) { setSource((s) => (s.kind === "gateway" ? { kind: "gecko", view: "trending" } : s)); } }; @@ -125,6 +129,13 @@ export function Dex() { const isSearch = source.kind === "gecko" && source.view === "token"; const isAddress = ADDRESS_RE.test(debouncedQuery); + // The query text only reaches the request for gateway sources and the token + // search view; keying other views on it would refetch a byte-identical + // listing (and spend GeckoTerminal budget) when leftover text sits in the + // box. Same for network/dexes, which gateway requests never send. + const effectiveQuery = + source.kind === "gateway" || isSearch ? debouncedQuery : ""; + const isGateway = source.kind === "gateway"; const enabled = !!server && source.kind !== "favorites" && @@ -144,9 +155,9 @@ export function Dex() { : source.kind === "gateway" ? source.connector : "favorites", - network, - debouncedQuery, - dexes.join(","), + isGateway ? "" : network, + effectiveQuery, + isGateway ? "" : dexes.join(","), page, ], queryFn: () => @@ -159,7 +170,7 @@ export function Dex() { // Same tab, same columns — filled in rather than em-dashed. source: source.connector === "orca" ? "orca" : "gateway", connector: source.connector, - query: debouncedQuery || undefined, + query: effectiveQuery || undefined, limit: PAGE_SIZE, page, } @@ -167,7 +178,7 @@ export function Dex() { source: "gecko", network, view: source.kind === "gecko" ? source.view : "trending", - query: isSearch ? debouncedQuery : undefined, + query: effectiveQuery || undefined, dexes, limit: PAGE_SIZE, page, @@ -184,11 +195,21 @@ export function Dex() { const { data: pastedPool } = useQuery({ queryKey: ["dex-pool-by-address", server, network, debouncedQuery], queryFn: () => - api - .getDexPoolByAddress(server!, debouncedQuery, network) - .catch(() => null as PoolSummary | null), + api.getDexPoolByAddress(server!, debouncedQuery, network).catch((e: Error) => { + // A pasted *token* address 404s here by design — the token search + // running alongside is the lookup that resolves it — so not-found is a + // soft null, not an error. Everything else (throttle 503, network) + // must stay an error: this cache key is shared with DexPool, and a + // null recorded as success renders its "No pool at …" dead-end there + // with the Retry button hidden. + if (/pool not found|request failed: 404/i.test(e.message)) { + return null as PoolSummary | null; + } + throw e; + }), enabled: !!server && isSearch && isAddress, staleTime: POOL_STALE_MS, + retry: false, }); const favoriteAddresses = useMemo( @@ -201,14 +222,72 @@ export function Dex() { isFetching: favoritesFetching, dataUpdatedAt: favoritesUpdatedAt, } = useQuery({ - queryKey: ["dex-favorites", server, network, favoriteAddresses.join(",")], - queryFn: () => - api.getDexPoolsByAddress(server!, network, favoriteAddresses), + // The address list is deliberately NOT part of the key: un-starring must + // remove a row the client already has without spending the shared + // GeckoTerminal budget on a refetch. The queryFn reads the current list + // from its closure, un-starred rows are filtered out client-side below, + // and only a *new* address (absent from the cached page) invalidates. + queryKey: ["dex-favorites", server, network], + queryFn: async () => { + // The endpoint takes at most 30 addresses per request, so favorites + // beyond that are fetched in extra batches instead of silently dropped. + const batches: string[][] = []; + for ( + let start = 0; + start < favoriteAddresses.length; + start += FAVORITES_BATCH_SIZE + ) { + batches.push( + favoriteAddresses.slice(start, start + FAVORITES_BATCH_SIZE), + ); + } + const pages = await Promise.all( + batches.map((batch) => + api.getDexPoolsByAddress(server!, network, batch), + ), + ); + return { + pools: pages.flatMap((page) => page.pools), + has_more: false, + // A throttled batch wins the merge: it is the one whose missing rows + // the empty-state message has to explain, whichever order it landed in. + upstream: + pages.find((page) => page.upstream?.throttled_request)?.upstream ?? + pages.at(-1)?.upstream, + }; + }, enabled: !!server && source.kind === "favorites" && !!favoriteAddresses.length, staleTime: POOL_STALE_MS, }); - const favoritePools = favoritePage?.pools ?? []; + // Un-starring shrinks this list without touching the query, so the row + // disappears with zero network requests. + const favoritePools = useMemo( + () => + (favoritePage?.pools ?? []).filter((p) => + favoriteAddresses.includes(p.address), + ), + [favoritePage, favoriteAddresses], + ); + + // Starring a pool the cached page has never seen is the one favorites change + // that genuinely needs upstream data. Reading the cache directly (rather + // than depending on `favoritePage`) keeps a throttled batch that dropped a + // row from re-triggering this on every response. + useEffect(() => { + const cached = queryClient.getQueryData<{ pools: PoolSummary[] }>([ + "dex-favorites", + server, + network, + ]); + if (!cached) return; // first fetch happens on its own + const known = new Set(cached.pools.map((p) => p.address)); + if (favoriteAddresses.some((a) => !known.has(a))) { + queryClient.invalidateQueries({ + queryKey: ["dex-favorites", server, network], + }); + } + }, [favoriteAddresses, server, network, queryClient]); // A response knows it was throttled before the next poll of /dex/upstream does, // and it is the response whose empty table the user is looking at. @@ -309,8 +388,9 @@ export function Dex() { )} - {/* The pasted-pool answer above already covers an empty search result — - showing "No pools found" under it would contradict what's right above it. */} + {/* A pasted address that resolved to a pool IS the result: an empty + token-search table saying "No pools found" under it would deny the + row right above. */} {!(pastedPool && !pools.length) && ( Object.entries(tokenVerdicts ?? {}) .filter(([, verdict]) => verdict === "symbol_taken" || verdict === "failed") - .map(([address]) => address), - [tokenVerdicts], + .map(([tokenAddress]) => { + const ticker = + tokenAddress === pool?.base_token_address + ? pool?.base_symbol + : tokenAddress === pool?.quote_token_address + ? pool?.quote_symbol + : undefined; + return { + address: tokenAddress, + symbol: ticker && ticker !== "???" ? ticker : undefined, + }; + }), + [tokenVerdicts, pool], ); + // The collision the last add ran into: who holds the ticker, so the banner + // can ask "replace it?" instead of failing with the same verdict again. + const [tokenConflict, setTokenConflict] = useState<{ + address: string; + symbol?: string; + holder: DexTokenConflict; + } | null>(null); + + const addTokenMutation = useMutation({ + mutationFn: (vars: { address: string; symbol?: string; replace?: boolean }) => + api.addDexToken(server!, network, vars.address, vars.symbol, vars.replace), + onSuccess: (data, vars) => { + if (data.verdict === "added" || data.verdict === "listed") { + setTokenConflict(null); + // Re-runs the ensure (its verdicts now read `listed`, dropping the + // banner) and refetches the portfolio the balance panels read from. + queryClient.invalidateQueries({ + queryKey: ["dex-ensure-tokens", server, network, pool?.address], + }); + queryClient.invalidateQueries({ queryKey: ["portfolio", server] }); + } else if (data.verdict === "symbol_taken" && data.conflict) { + setTokenConflict({ + address: vars.address, + symbol: vars.symbol, + holder: data.conflict, + }); + } + }, + }); + const { data: venues = [] } = useQuery({ queryKey: ["venues", server], queryFn: () => api.getVenues(server!), @@ -222,6 +271,15 @@ export function DexPool() { const symbolPair = baseSymbol && quoteSymbol ? `${baseSymbol}-${quoteSymbol}` : ""; + // WS for executor data, same subscription CreateExecutor holds: without it + // the shared-socket bridge has no executors frames to fan out here, and the + // pair queries below (staleTime 30s, no polling) never see a stop or a fill. + const wsChannels = useMemo( + () => (server ? [`executors:${server}`] : []), + [server], + ); + useCondorWebSocket(wsChannels, server ?? null); + const { executors, overlays, positions, isLoadingPositions } = useMainControllerData(server ?? null, network, pair, { altPair: symbolPair, @@ -331,7 +389,8 @@ export function DexPool() { {/* Gateway lists a token by ticker, so a pool whose token shares a ticker with one already on the list cannot be registered — and the only symptom - is a balance that reads 0 for a wallet that is not empty. */} + is a balance that reads 0 for a wallet that is not empty. The button is + the fix in place: retry a failed save, or replace the ticker's holder. */} {unlistedTokens.length > 0 && (
- Balances for {unlistedTokens.map((a) => `${a.slice(0, 6)}…`).join(", ")}{" "} - will read 0 — add {unlistedTokens.length === 1 ? "it" : "them"} under - Gateway tokens to size orders from this wallet. + Balances for{" "} + {unlistedTokens + .map((t) => t.symbol ?? `${t.address.slice(0, 6)}…`) + .join(", ")}{" "} + will read 0 until {unlistedTokens.length === 1 ? "it is" : "they are"}{" "} + added. + {unlistedTokens.map((t) => ( + + ))} + {tokenConflict && ( + + {tokenConflict.holder.symbol} already names{" "} + {tokenConflict.holder.address.slice(0, 6)}… on the list — replace + it with this pool's token? + + + )} + {!tokenConflict && + addTokenMutation.data?.verdict === "symbol_taken" && + !addTokenMutation.data.conflict && ( + + Its ticker is already used by another token on the list — resolve + it under Gateway tokens. + + )} + {addTokenMutation.isError && ( + + {addTokenMutation.error instanceof Error + ? addTokenMutation.error.message + : "Add failed"} + + )}
)} @@ -361,10 +472,12 @@ export function DexPool() {
{pairLabel} - {network} + {/* The address identifies the pool the way the ticker never can, so it + sits with the name instead of hiding at the far right. */} +
- +
@@ -372,7 +485,7 @@ export function DexPool() {
Interval - {INTERVALS.map((iv) => ( + {DEX_INTERVALS.map((iv) => (
{successId && ( -
-
- -
-
- -
-
-

- {tab === "lp" ? "LP Position" : "Order"} Created -

-

- In {pairLabel} on {pool.dex_id} -

-
- Executor ID -
- {successId.slice(0, 12)}… - -
-
- -
-
+ setSuccessId(null)} + /> )} {createMutation.isError && ( -
- {(createMutation.error as Error).message} -
+ )}
); diff --git a/frontend/src/pages/Login.tsx b/frontend/src/pages/Login.tsx index b8d1bab75..214672ca0 100644 --- a/frontend/src/pages/Login.tsx +++ b/frontend/src/pages/Login.tsx @@ -4,12 +4,16 @@ import { useNavigate, useSearchParams } from "react-router-dom"; import { useAuth } from "@/lib/auth"; export function Login() { - const { isAuthenticated, loginWithToken } = useAuth(); + const { isAuthenticated, loginWithToken, loginLocal } = useAuth(); const navigate = useNavigate(); const [searchParams] = useSearchParams(); const [error, setError] = useState(""); const [loggingIn, setLoggingIn] = useState(false); const attempted = useRef(false); + // How this install logs in — asked once, before anything is rendered, so the + // "run /web in Telegram" card never flashes on a machine that has no Telegram. + const [mode, setMode] = useState(null); + const probed = useRef(false); // Where to land after login. Only allow internal paths to avoid open redirects. const rawRedirect = searchParams.get("redirect"); @@ -44,12 +48,44 @@ export function Login() { }); }, [searchParams, loginWithToken, navigate, redirectTo]); + // Local mode (FEAT-049): there is no login step. Ask the server how it + // authenticates and, if the answer is "locally", claim the session and go. + // Skipped entirely when a one-time ?token= is being redeemed above. + useEffect(() => { + if (searchParams.get("token") || probed.current) return; + probed.current = true; + + let cancelled = false; + fetch("/api/v1/auth/mode") + .then((res) => (res.ok ? res.json() : { mode: "telegram" })) + .then((data) => { + if (cancelled) return; + setMode(data?.mode ?? "telegram"); + if (data?.mode !== "local") return; + setLoggingIn(true); + return loginLocal() + .then(() => navigate(redirectTo, { replace: true })) + .catch((err) => { + setError(err instanceof Error ? err.message : "Login failed"); + setLoggingIn(false); + }); + }) + .catch(() => { + // The server is unreachable, not Telegram-less: show the normal card. + if (!cancelled) setMode("telegram"); + }); + + return () => { + cancelled = true; + }; + }, [searchParams, loginLocal, navigate, redirectTo]); + return (
Condor

Condor

- {loggingIn ? ( + {loggingIn || (mode === null && !searchParams.get("token")) ? (

Signing in...

diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index 325384ead..9e2b21113 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -7,6 +7,7 @@ import { ApiKeysSettings } from "@/components/settings/ApiKeysSettings"; import { CustomProvidersSettings } from "@/components/settings/CustomProvidersSettings"; import { GatewaySettings } from "@/components/settings/GatewaySettings"; import { ServersSettings } from "@/components/settings/ServersSettings"; +import { TelemetrySettings } from "@/components/settings/TelemetrySettings"; import { VoiceSettings } from "@/components/settings/VoiceSettings"; import { ADMIN_USERS_KEY, adminApi } from "@/lib/admin-api"; import { useAuth } from "@/lib/auth"; @@ -17,6 +18,7 @@ const TABS = [ { key: "keys", label: "Keys and Wallets" }, { key: "llm", label: "LLM Endpoints" }, { key: "voice", label: "Voice & AI" }, + { key: "privacy", label: "Privacy" }, ] as const; /** Admin-only, appended to TABS when the user turns out to be an admin (ARCH-177). */ @@ -80,6 +82,7 @@ export function Settings() { {tab === "keys" && } {tab === "llm" && } {tab === "voice" && } + {tab === "privacy" && } {tab === "admin" && }
); diff --git a/handlers/agents/_shared.py b/handlers/agents/_shared.py index d51a29f42..90a38f846 100644 --- a/handlers/agents/_shared.py +++ b/handlers/agents/_shared.py @@ -1,67 +1,55 @@ -"""Constants and MCP config loader for agent sessions.""" +"""Telegram-side agent helpers, plus re-exports of the moved core (ARCH-190). + +What remains here is genuinely Telegram-flavored: the chat↔agent binding and +conversation bookkeeping. The platform-neutral foundations moved down into +``condor/`` — the model catalog (:mod:`condor.llm.options`), the MCP toolset +builders (:mod:`condor.runtime.toolsets`), the danger classification +(:mod:`condor.runtime.danger`) and the context builders +(:mod:`condor.runtime.context`) — and are re-exported below so existing +handler imports keep working. New code should import from the new homes. +""" import hashlib -import json import logging -import os -from pathlib import Path -from typing import Any -from condor.acp.client import bot_process_marker -from condor.memory.paths import CHAT_SLUG +# Re-exports for Telegram callers (ARCH-190) — the implementations live in +# condor/ now; condor/ itself must never import back through this module. +from condor.llm.options import ( # noqa: F401 + AGENT_OPTIONS, + DEFAULT_AGENT, + RECOMMENDED_AGENT, + selectable_agent_options, +) +from condor.runtime.context import ( # noqa: F401 + _TELEGRAM_FORMATTING, + _WEB_FORMATTING, + COMPACT_CONTEXT_TEMPLATE, + COMPACT_PROMPT_AUTO, + COMPACT_PROMPT_CUSTOM_TEMPLATE, + build_agent_context, + build_initial_context, + platform_formatting, +) +from condor.runtime.danger import ( # noqa: F401 + BLOCKED_TOOLS, + DANGEROUS_BOT_ACTIONS, + DANGEROUS_CLMM_ACTIONS, + DANGEROUS_EXECUTOR_ACTIONS, + DANGEROUS_SWAP_ACTIONS, + DANGEROUS_TOOLS, + is_dangerous_tool_call, + tool_call_input, + tool_call_name, +) +from condor.runtime.toolsets import ( # noqa: F401 + _bot_id_args, + build_mcp_servers_for_session, + get_project_dir, +) log = logging.getLogger(__name__) -def _chat_agent(): - """The default agent's record — Condor's ``agents/condor/AGENT.md``. - - There is one loader and one frontmatter schema (FEAT-033): the chat is read - by ``AgentStore`` exactly like a specialist is. Returns ``None`` only if the - file is missing or unreadable, which callers treat as "no instructions" - rather than as a startup failure. - """ - from condor.agents.agent import AgentStore - - return AgentStore().get(CHAT_SLUG) - - -AGENT_OPTIONS: dict[str, dict[str, Any]] = { - "claude-code": {"label": "Claude Code"}, - "claude-acp:opus": {"label": "Claude (ACP) — Opus"}, - "claude-acp:sonnet": {"label": "Claude (ACP) — Sonnet"}, - "gemini": {"label": "Gemini CLI"}, - "copilot": {"label": "GitHub Copilot CLI"}, - "codex": {"label": "ChatGPT Codex"}, - "ollama:": {"label": "Ollama — Default Model"}, - "lmstudio:": {"label": "LM Studio — Default Model"}, - # Sentinels — these open a picker instead of being a selectable model, so - # they are NOT valid agent keys. `picker` marks them for surfaces that - # render AGENT_OPTIONS as a flat list of choices (the web dashboard's model - # dropdown), which would otherwise offer a key that fails at session start. - # - # OpenRouter: stored agent_llm becomes "openrouter:". - "openrouter:": {"label": "OpenRouter — Pick Model", "picker": True}, - # Custom endpoints: the user's saved OpenAI-compatible endpoints live in - # preferences (condor/preferences.py, shared with the web dashboard) and - # the stored agent_llm becomes "custom@:". - "custom:": {"label": "Custom — OpenAI-compatible API", "picker": True}, -} - - -# What a fresh install should pick unless it has a reason not to: the ACP bridge -# is the best-supported path, and Sonnet is the balance of capability and cost -# most users want. Surfaces that rank the catalog (the `make pick-model` wizard) -# float this key to the top and mark it; it is deliberately NOT part of the -# `label`, which is reused as a plain display name all over the bot. -RECOMMENDED_AGENT = "claude-acp:sonnet" - - -def selectable_agent_options() -> dict[str, dict[str, Any]]: - """AGENT_OPTIONS minus the picker sentinels — every key here is startable.""" - return {k: v for k, v in AGENT_OPTIONS.items() if not v.get("picker")} - - def bind_chat_to_agent(user_data: dict | None, agent_slug: str) -> None: """Record (or clear) the specialist this chat talks to. @@ -190,642 +178,3 @@ def find_conversation(metas: list, token: str): must treat that as a normal outcome and say so rather than failing. """ return next((m for m in metas if conversation_token(m.id) == token), None) - - -def _default_agent() -> str: - """Resolve the fallback agent_key for users who haven't picked a model. - - Precedence: ``CONDOR_DEFAULT_AGENT`` env > Condor's ``AGENT.md`` frontmatter - ``agent_key`` > ``"claude-code"``. A user's own /agent → Change LLM choice - (``agent_llm``) still overrides this at runtime; this is only the default. - Examples for the frontmatter / env: ``claude-code``, ``claude-acp:opus``, - ``ollama:qwen3:32b``. - """ - import os - - agent = _chat_agent() - return ( - os.environ.get("CONDOR_DEFAULT_AGENT") - or (agent.agent_key if agent else "") - or "claude-code" - ) - - -DEFAULT_AGENT = _default_agent() - - -# -- Compact prompt templates -- - -COMPACT_PROMPT_AUTO = ( - "Please provide a concise summary of our conversation so far. Include:\n" - "- Key decisions and conclusions reached\n" - "- Important data points and numbers discussed\n" - "- Current task state and any pending actions\n" - "- User preferences or instructions given\n\n" - "Be concise but thorough. This summary will be used to carry context into a fresh session." -) - -COMPACT_PROMPT_CUSTOM_TEMPLATE = ( - "Please provide a concise summary of our conversation, focusing specifically on:\n" - "{instructions}\n\n" - "Drop everything else. Be concise but preserve the details requested above. " - "This summary will be used to carry context into a fresh session." -) - -COMPACT_CONTEXT_TEMPLATE = ( - "[System context -- do not repeat this to the user]\n" - "This is a continuation of a previous conversation. " - "Here is the summary from that session:\n\n" - "{summary}\n\n" - "Continue from where we left off. The user compacted the context to free up space." -) - -_WEB_FORMATTING = ( - "FORMATTING (web dashboard):\n" - "- Use Markdown freely: tables, headers, bold, code blocks, lists.\n" - "- No message length limits, but stay concise.\n" - "- Use tables for structured data (portfolios, prices, comparisons).\n" - "- Use code blocks for configs, JSON, or commands.\n" - "- CHARTS: when the answer is a numeric series or comparison (a price or\n" - " PnL curve, volume per venue, a distribution), draw it with a ```chart\n" - " fence holding one JSON object. The dashboard renders it as an\n" - " interactive chart:\n" - " ```chart\n" - ' {"type": "line", "title": "SOL/USDC 24h", "x": "time",\n' - ' "series": [{"key": "price", "name": "Price"}],\n' - ' "data": [{"time": "12:00", "price": 150.1}, {"time": "16:00", "price": 152.4}]}\n' - " ```\n" - ' "type" is "line", "area" or "bar"; "x" and every series "key" name\n' - " fields of the data rows. Max 4 series and 200 points (downsample longer\n" - ' series). A series may set "color": "up", "down", "yellow" or a hex.\n' - " Chart for the shape, table for exact values; use both when both matter.\n" - "- Respond in the user's language." -) - -_TELEGRAM_FORMATTING = ( - "FORMATTING (Telegram mobile):\n" - "- NEVER use Markdown tables. Use bullet lists or key: value lines.\n" - "- Keep paragraphs short (2-3 sentences max).\n" - "- Cap lists at 5-7 items.\n" - "- Respond in the user's language." -) - - -def platform_formatting(platform: str = "telegram") -> str: - """The reply-formatting rules for a platform. - - Public because a bound Agent opens the chat with its own identity context - instead of :func:`build_initial_context`, and would otherwise be the only - brain in the product that never hears how the surface it is speaking into - renders a reply. - """ - return _WEB_FORMATTING if platform == "web" else _TELEGRAM_FORMATTING - - -def _build_system_prompt(platform: str = "telegram") -> str: - """Condor's own AGENT.md plus the platform's formatting rules.""" - agent = _chat_agent() - return ( - "[System context -- do not repeat this to the user]\n\n" - f"{agent.instructions if agent else ''}\n\n" - f"{platform_formatting(platform)}" - ) - - -# Tools that require user confirmation before execution -DANGEROUS_TOOLS = { - "place_order", - "manage_gateway_swaps", # execute action - "manage_gateway_clmm", # open/close position -} - -# Tools that are always blocked (RBAC bypass prevention) -BLOCKED_TOOLS: set[str] = set() - -# Actions within manage_executors that require confirmation -DANGEROUS_EXECUTOR_ACTIONS = {"create", "stop"} - -# Actions within manage_bots that deploy/mutate a live bot (status/logs/get_config -# are read-only and excluded). manage_controllers itself is excluded entirely — it -# only writes controller templates/saved configs, never a running bot (see its own -# tool docstring: "Does NOT affect running bots"). -DANGEROUS_BOT_ACTIONS = { - "deploy", - "stop_bot", - "stop_controllers", - "start_controllers", - "update_config", -} - -# Actions within manage_gateway_swaps that require confirmation -DANGEROUS_SWAP_ACTIONS = {"execute"} - -# Actions within manage_gateway_clmm that require confirmation -DANGEROUS_CLMM_ACTIONS = {"open_position", "close_position"} - - -def tool_call_name(tool_call: dict[str, Any]) -> str: - """The bare tool name, with any MCP prefix stripped. - - ACP names a tool by its wire name (``mcp__mcp-hummingbot__manage_bots``) - where the local surfaces use the bare one, so everything that dispatches on - a tool name — the danger list, the risk gate, the confirmation summary — - has to normalize identically or they disagree about the same call. - """ - raw_name = tool_call.get("tool", "") or tool_call.get("title", "") - return raw_name.rsplit("__", 1)[-1] if "__" in raw_name else raw_name - - -def tool_call_input(tool_call: dict[str, Any]) -> dict[str, Any] | None: - """A tool call's arguments as a mapping, or ``None`` when unreadable. - - ``None`` means "I cannot tell what this call does", and every caller has to - fail closed on it: an action-gated tool whose action can't be read is - treated as dangerous rather than waved through (SEC-093). Callers must not - reach for other spellings of the arguments — the ACP wire's ``rawInput`` is - translated once, at the boundary in :func:`condor.acp.client.normalize_tool_call`. - - A JSON string is parsed: OpenAI-compatible providers deliver tool arguments - that way. - """ - args = tool_call.get("input") - if isinstance(args, dict): - return args - if isinstance(args, str) and args.strip(): - try: - parsed = json.loads(args) - except json.JSONDecodeError: - return None - return parsed if isinstance(parsed, dict) else None - return None - - -def _has_dangerous_action( - tool_call: dict[str, Any], dangerous_actions: set[str] -) -> bool: - """Whether an action-gated tool call selects one of its dangerous actions. - - Fails closed (SEC-093): unreadable arguments, or a missing/non-string - ``action``, count as dangerous. ``action`` is required by every one of - these tools, so an unreadable one is never a benign read — it is a call we - failed to understand, and those belong in front of a human. - """ - input_data = tool_call_input(tool_call) - if input_data is None: - return True - action = input_data.get("action") - if not isinstance(action, str) or not action: - return True - return action in dangerous_actions - - -def is_dangerous_tool_call(tool_call: dict[str, Any]) -> bool: - """Check if a tool call requires user confirmation.""" - tool_name = tool_call_name(tool_call) - - # Direct dangerous tools - if tool_name in DANGEROUS_TOOLS: - # For manage_gateway_swaps, only "execute" action is dangerous - if tool_name == "manage_gateway_swaps": - return _has_dangerous_action(tool_call, DANGEROUS_SWAP_ACTIONS) - - # For manage_gateway_clmm, only open/close are dangerous - if tool_name == "manage_gateway_clmm": - return _has_dangerous_action(tool_call, DANGEROUS_CLMM_ACTIONS) - - return True - - # manage_executors with create/stop actions - if tool_name == "manage_executors": - return _has_dangerous_action(tool_call, DANGEROUS_EXECUTOR_ACTIONS) - - # manage_bots with deploy/stop/update actions (a bot deploy places real - # capital via a controller, a different path than manage_executors) - if tool_name == "manage_bots": - return _has_dangerous_action(tool_call, DANGEROUS_BOT_ACTIONS) - - return False - - -def get_project_dir() -> str: - """Get the condor project root directory (where .mcp.json lives). - - The ACP agent auto-discovers stdio MCP servers from .mcp.json in the cwd, - so we just need to point it at the project root. - """ - return str(Path(__file__).parent.parent.parent) - - -def _bot_token() -> str: - """This bot's Telegram token, read at call time. - - Goes through ``utils.config`` for its ``load_dotenv()`` side effect, so the - marker built here is derived from the same token ``main.py`` hands the - reaper — a mismatch would silently strand orphaned subprocess trees. - """ - import utils.config # noqa: F401 (imported for load_dotenv()) - - return os.environ.get("TELEGRAM_TOKEN", "") - - -def _bot_id_args() -> list[str]: - """``--bot-id `` for an MCP subprocess, or nothing without a token. - - The digest is what the startup reaper (``reap_stale_acp_trees``) seeds on to - find subprocess trees orphaned by a previous crash. It replaced the raw - token, which used to sit here in clear text (SEC-095). Both MCP servers - parse their argv with ``parse_known_args``, so the flag is inert to them — - it exists purely to be visible in ``ps``. - """ - marker = bot_process_marker(_bot_token()) - return ["--bot-id", marker] if marker else [] - - -def _env_entries(**values: Any) -> list[dict[str, str]]: - """ACP ``env`` entries (``{"name", "value"}``), skipping empty values. - - This is the channel every secret an MCP subprocess needs must travel on: the - ACP bridge and the pydantic-ai stdio client both overlay these onto the - child's inherited environment, which — unlike argv — no other local user can - read out of ``ps``. - - Values are coerced to ``str``: YAML loads an unquoted numeric credential as - int (e.g. ``password: 123``), and an int in the environment mapping breaks - subprocess spawning — for pydantic-ai backends (lmstudio:/ollama:/ - openrouter:) it surfaces as a StdioServerParameters validation error. - """ - entries = ( - (name, str(value)) for name, value in values.items() if value is not None - ) - return [{"name": name, "value": value} for name, value in entries if value] - - -def _condor_mcp_args( - chat_id: int | str, - user_id: int, - agent_slug: str | None = None, - server_name: str | None = None, - delegate_worker: bool = False, -) -> list[str]: - """Build CLI args for the condor MCP subprocess. - - ``delegate_worker`` marks a *background Condor worker* — the detached session - ``delegate`` starts (FEAT-032). The chat and that worker share one agent - record, so the flag is what tells the subprocess which seat it is sitting in. - - The bot token travels in the server's ``env``, never here: argv is - world-readable via ``ps`` (SEC-095). What argv carries instead is - ``--bot-id``, the token's non-secret digest, which is how the startup reaper - recognizes our own leaked subprocess trees. - """ - # MCP server expects int chat_id. For web sessions (string keys like "web_42"), - # use user_id instead — in Telegram DMs, chat_id == user_id anyway. - effective_chat_id = chat_id if isinstance(chat_id, int) else user_id - args = [ - "--chat-id", - str(effective_chat_id), - "--user-id", - str(user_id), - ] - args.extend(_bot_id_args()) - if agent_slug: - args.extend(["--agent-slug", str(agent_slug)]) - if server_name: - args.extend(["--server-name", str(server_name)]) - if delegate_worker: - args.append("--delegate-worker") - return args - - -def _hummingbot_mcp_args(server: dict[str, Any], server_name: str) -> list[str]: - """Build CLI args for the hummingbot MCP subprocess. - - Only non-secret coordinates travel here — the API username/password go in - the server's ``env`` instead, since argv is world-readable via ``ps`` - (SEC-095). Every element is a string: YAML loads unquoted numerics as int - (e.g. ``port: 8000``), and pydantic-ai's StdioServerParameters rejects - non-string args when starting LM Studio / other local-model sessions. - """ - api_url = f"http://{server['host']}:{server['port']}" - return [ - "run", - "python", - "-m", - "mcp_servers.hummingbot_api", - "--url", - api_url, - "--server-name", - str(server_name), - ] + _bot_id_args() - - -def build_mcp_servers_for_session( - user_id: int, - chat_id: int | str, - user_data: dict | None = None, - server_name: str | None = None, - agent_slug: str | None = None, - delegate_worker: bool = False, -) -> list[dict[str, Any]]: - """Build dynamic MCP server configs for an agent session. - - Returns ACP-format mcpServers that override the static .mcp.json entries by - name. Always includes the condor MCP server; hummingbot is added when a - valid server can be resolved for the user. - - ``server_name`` pins the run to that Condor server (an Agent with - ``server_required``); when omitted, the chat's ambient server is resolved - from the user's preferences, then from the first accessible server. Each - candidate must exist *and* be reachable by ``user_id`` — a name this user - has no grant on resolves to the next candidate, never to its credentials. - - ``agent_slug`` scopes the condor MCP tools' memory/skills to that Agent's - own stores (``agents/{slug}/``). Without it the tools target the chat - condor's stores — correct for chat sessions, wrong for an Agent run: a - serverless consult/tick would silently read and write the CHAT's memory - and skills instead of the Agent's own (e.g. an agent unable to find its - inherited ``routine_cookbook`` skill). - - ``delegate_worker`` marks a background Condor delegation (FEAT-032) so the - subprocess picks up the worker framing and refuses to delegate again. - """ - from config_manager import get_config_manager, get_effective_server - - cm = get_config_manager() - - # Resolve which hummingbot server to use (explicit override > user - # preferences). Every candidate is held to existence *and* reach, because - # the name that comes out of here decides whose API credentials go into the - # subprocess env below. ``chat_id`` is not a principal — ``chat_defaults`` - # is a global map keyed by chat, so a caller who can name someone else's - # chat could name their server too (SEC-178, the SEC-164 shape). The subject - # is always ``user_id``, the authenticated owner of the run; the same - # predicate guards TickEngine._resolve_server. - def usable(name: str | None) -> bool: - return bool(name and cm.get_server(name)) and cm.has_server_access( - user_id, name - ) - - def candidates(): - # Lazy on purpose: resolving the chat default writes back into - # ``user_data``, so it must not run when a pin already answered. - yield server_name - yield get_effective_server(chat_id, user_data) - accessible = cm.get_accessible_servers(user_id) - yield accessible[0] if accessible else None - - server_name = next((name for name in candidates() if usable(name)), None) - - # Condor MCP -- runs as stdio subprocess, tools work locally without TCP bridge - # Pass resolved server_name so start_agent uses the correct server - condor = { - "name": "condor", - "command": "uv", - "args": ["run", "python", "-m", "mcp_servers.condor"] - + _condor_mcp_args( - chat_id, - user_id, - agent_slug, - server_name=server_name, - delegate_worker=delegate_worker, - ), - "env": _env_entries(TELEGRAM_BOT_TOKEN=_bot_token()), - } - - if not server_name: - log.warning( - "No accessible server for user %s (chat %s) — " - "agent will start without mcp-hummingbot", - user_id, - chat_id, - ) - return [condor] - - server = cm.get_server(server_name) - if not server: - log.warning( - "Server '%s' resolved for user %s but not found in servers config — " - "agent will start without mcp-hummingbot", - server_name, - user_id, - ) - return [condor] - - # Credentials go in env, not argv: the API username/password used to sit on - # the command line, where any local `ps` recovered them (SEC-095). The - # non-secret coordinates (url, server name) stay on argv, where they make a - # running subprocess identifiable. - mcp_hummingbot = { - "name": "mcp-hummingbot", - "command": "uv", - "args": _hummingbot_mcp_args(server, server_name), - "env": _env_entries( - HUMMINGBOT_API_USERNAME=server["username"], - HUMMINGBOT_API_PASSWORD=server["password"], - ), - } - - return [mcp_hummingbot, condor] - - -def build_initial_context( - user_id: int, - chat_id: int | str, - user_data: dict | None = None, - agent_key: str | None = None, - platform: str = "telegram", - server_name: str | None = None, -) -> str: - """Build an initial context prompt telling the agent about server, permissions, and formatting rules.""" - from condor.acp.pydantic_ai_client import is_pydantic_ai_model - from config_manager import ( - ServerPermission, - get_config_manager, - get_effective_server, - ) - - cm = get_config_manager() - - # System prompt: Condor's own AGENT.md + platform formatting - system_prompt = _build_system_prompt(platform) - sections: list[str] = [system_prompt] - - # Resolve active server (explicit override > user preferences) - active_name = server_name or get_effective_server(chat_id, user_data) - accessible = cm.get_accessible_servers(user_id) - if not active_name: - active_name = accessible[0] if accessible else None - - if active_name: - # Build server list with permissions (no credentials needed — MCP is pre-configured) - server_lines: list[str] = [] - for name in accessible: - server = cm.get_server(name) - if not server: - continue - perm = cm.get_server_permission(user_id, name) - perm_label = perm.value.upper() if perm else "UNKNOWN" - active_tag = " (active)" if name == active_name else "" - server_lines.append(f"- {name}{active_tag} [{perm_label}]") - - # Get active server permission for enforcement - active_perm = cm.get_server_permission(user_id, active_name) - active_perm_label = active_perm.value.upper() if active_perm else "UNKNOWN" - - # For ACP agents (Claude Code): instruct them to preload MCP tools via ToolSearch - # Pydantic-ai agents get tools directly, no preload needed - tool_preload_hint = "" - if agent_key and not is_pydantic_ai_model(agent_key): - mcp_tools = [ - "mcp__mcp-hummingbot__get_market_data", - "mcp__mcp-hummingbot__get_portfolio_overview", - "mcp__mcp-hummingbot__manage_executors", - "mcp__mcp-hummingbot__manage_bots", - "mcp__mcp-hummingbot__manage_controllers", - "mcp__mcp-hummingbot__explore_dex_pools", - "mcp__mcp-hummingbot__explore_geckoterminal", - "mcp__mcp-hummingbot__manage_gateway_swaps", - "mcp__mcp-hummingbot__manage_gateway_config", - "mcp__mcp-hummingbot__manage_gateway_container", - "mcp__mcp-hummingbot__search_history", - "mcp__mcp-hummingbot__set_account_position_mode_and_leverage", - "mcp__condor__manage_routines", - "mcp__condor__manage_servers", - "mcp__condor__get_user_context", - "mcp__condor__manage_trading_agent", - "mcp__condor__trading_agent_journal_read", - "mcp__condor__trading_agent_journal_write", - "mcp__condor__send_notification", - "mcp__condor__manage_memory", - "mcp__condor__manage_skill", - ] - tool_preload_hint = ( - "IMPORTANT: At the very start of the session (before your first response), " - "load ALL MCP tools in a single ToolSearch call:\n" - f'ToolSearch(query="select:{",".join(mcp_tools)}")\n' - "This avoids repeated ToolSearch calls that waste context tokens. " - "Do this silently without telling the user." - ) - - # Build server info section - server_info = [ - f"Active server: {active_name} (permission: {active_perm_label})", - "The MCP server is already connected to this server. Do NOT call configure_server — it is pre-configured.", - "", - ] - - if tool_preload_hint: - server_info.extend([tool_preload_hint, ""]) - - server_info.extend( - [ - "Available servers:", - *server_lines, - "", - "Server switching is not supported mid-session. If the user wants a different server, " - "tell them to start a new session with that server selected.", - "", - "Permission rules:", - f"- Your current permission on '{active_name}' is {active_perm_label}.", - "- OWNER: Full access including trading operations and server management.", - "- TRADER: Can trade, view balances, and manage own settings.", - "- Enforce the permission level for this server.", - ] - ) - - sections.append("\n".join(server_info)) - - # User memory index — what the chat assistant remembers about this user. This - # store is the chat's own (FEAT-003), not shared with the user's trading - # agents. Inject only the index; bodies are read on demand via - # manage_memory(action="read"). Nothing injected for new users. - try: - from condor.memory import MemoryStore - - memory_index = MemoryStore(user_id).list_index() - if memory_index: - sections.append( - "[USER MEMORY — what you remember about this user]\n" - "Consider this before responding. Read a full memory with " - 'manage_memory(action="read", name="..."). When you learn something ' - 'new and stable about the user, save it with manage_memory(action="write", ...).\n\n' - f"{memory_index}" - ) - except Exception: - pass # Memory is advisory — never block session start on it. - - # Skills index — read-only playbooks the chat assistant ships (know-how + - # steps). Skills are general to the assistant, not learned per user; inject - # only the index, bodies are read on demand via manage_skill(action="read"). - # Nothing injected when the assistant ships none. - try: - from condor.memory import SkillStore - - skills_index = SkillStore().list_index() - if skills_index: - sections.append( - "[SKILLS — check here BEFORE handling a known flow with raw tools]\n" - "Read-only playbooks shipped with the assistant. Before using raw " - "tools for a flow below, read its playbook with " - 'manage_skill(action="read", name="...") and follow the steps — ' - "don't re-derive or hand-roll what a playbook already covers.\n\n" - f"{skills_index}" - ) - except Exception: - pass # Skills are advisory — never block session start on them. - - # Agents index — domain Agents condor can consult (FEAT: coordinator model). - # condor delegates domain work via consult(...) instead of holding the domain's - # tools/context itself. EVERY agent is listed: one missing from this index can - # never be routed to. Nothing injected only when none exist. - try: - from condor.agents.agent import AgentStore - - # The coordinator is not among its own consultees (FEAT-033). - agents_index = AgentStore().list_index(exclude={CHAT_SLUG}) - if agents_index: - sections.append( - "[AGENTS — consult these BEFORE doing domain work with raw tools]\n" - "You are a coordinator. If a request falls in an agent's domain " - "below, delegate it with " - 'consult(agent="", task="...", context="...") instead of ' - "driving the domain's raw tools yourself — the agent has the focused " - "tools and domain memory. Relay a concise summary of its answer.\n\n" - f"{agents_index}" - ) - except Exception: - pass # The agents index is advisory — never block session start on it. - - return "\n\n".join(sections) - - -def build_agent_context( - agent: "Agent", # noqa: F821 — condor.agents.agent.Agent - user_id: int, - task: str, - context: str = "", -) -> str: - """Assemble the prompt for an Agent consult. - - Mirrors :func:`build_initial_context` but for a worker run: the Agent's own - instructions become the system prompt, its domain-scoped memory/skills indexes - are injected (keyed by the Agent slug, FEAT-003 — the shared brain), and the - consult task is appended last. Read on demand via manage_memory/manage_skill - inside the run. - - Those indexes come from :func:`~condor.memory.domain_context`, the same - builder ``binding.agent_identity_context`` composes, so this Agent carries - identical instructions about its own memory whether it is consulted or - chatted with (ARCH-099). - """ - from condor.memory import domain_context - - sections: list[str] = [agent.instructions] - sections.extend(domain_context(agent.slug, user_id)) - - consult = f"[CONSULT REQUEST]\n{task}" - if context: - consult += f"\n\n[CONTEXT FROM CONDOR]\n{context}" - sections.append(consult) - - return "\n\n".join(sections) diff --git a/handlers/agents/confirmation.py b/handlers/agents/confirmation.py index 9ef63f4fb..0f30ca3fc 100644 --- a/handlers/agents/confirmation.py +++ b/handlers/agents/confirmation.py @@ -9,13 +9,11 @@ import logging from typing import Any -from telegram import Bot, InlineKeyboardButton, InlineKeyboardMarkup +from telegram import Bot -from condor.runtime.confirmations import ( - CONFIRMATION_TIMEOUT, - PendingConfirmation, - get_registry, -) +from condor.runtime.channels import TelegramChannel # noqa: F401 (re-export) +from condor.runtime.confirmations import CONFIRMATION_TIMEOUT, get_registry +from condor.runtime.danger import format_tool_summary # noqa: F401 (re-export) log = logging.getLogger(__name__) @@ -28,112 +26,11 @@ ] -def format_tool_summary(tool_call: dict[str, Any]) -> str: - """Format a tool call into a human-readable summary for the confirmation message.""" - from handlers.agents._shared import tool_call_input, tool_call_name - - # The bare name, or an ACP call (``mcp__mcp-hummingbot__manage_bots``) would - # match none of the branches below and be approved as an opaque string. - tool_name = tool_call_name(tool_call) or "Unknown" - input_data = tool_call_input(tool_call) - if input_data is None: - # The gate sends unreadable calls here on purpose (SEC-093). Say so, - # rather than rendering a row of "?" that looks like a parsed call. - return f"{tool_name} (arguments could not be read)" - - if tool_name == "place_order": - side = input_data.get("trade_type", "?") - pair = input_data.get("trading_pair", "?") - amount = input_data.get("amount", "?") - order_type = input_data.get("order_type", "MARKET") - price = input_data.get("price", "") - connector = input_data.get("connector_name", "?") - summary = f"{side} {amount} {pair} ({order_type})" - if price: - summary += f" @ {price}" - summary += f" on {connector}" - return summary - - if tool_name == "manage_executors": - action = input_data.get("action", "?") - exec_type = input_data.get("executor_type", "") - exec_id = input_data.get("executor_id", "") - if action == "create" and exec_type: - config = input_data.get("executor_config", {}) - pair = config.get("trading_pair", "?") - return f"Create {exec_type} on {pair}" - if action == "stop" and exec_id: - return f"Stop executor {exec_id[:12]}..." - return f"Executor: {action}" - - if tool_name == "manage_bots": - action = input_data.get("action", "?") - bot_name = input_data.get("bot_name", "?") - if action == "deploy": - controllers = input_data.get("controllers_config", []) - return f"Deploy bot '{bot_name}' with controllers {controllers}" - if action == "update_config": - config_name = input_data.get("config_name", "?") - return f"Update config '{config_name}' on bot '{bot_name}'" - return f"Bot '{bot_name}': {action}" - - if tool_name == "manage_gateway_swaps": - action = input_data.get("action", "?") - pair = input_data.get("trading_pair", "?") - side = input_data.get("side", "?") - amount = input_data.get("amount", "?") - return f"Swap {side} {amount} {pair}" - - if tool_name == "manage_gateway_clmm": - action = input_data.get("action", "?") - if action == "open_position": - return "Open LP position" - if action == "close_position": - return "Close LP position" - return f"CLMM: {action}" - - # Generic fallback - return tool_name - - -# Backwards-compatible alias for the pre-registry name. +# Backwards-compatible alias for the pre-registry name; the implementation +# moved to condor.runtime.danger (ARCH-190). _format_tool_summary = format_tool_summary -class TelegramChannel: - """Renders a pending confirmation as an inline keyboard in a chat.""" - - def __init__(self, bot: Bot, chat_id: int): - self._bot = bot - self._chat_id = chat_id - - async def deliver(self, pending: PendingConfirmation) -> None: - keyboard = InlineKeyboardMarkup( - [ - [ - InlineKeyboardButton( - "Approve", callback_data=f"agent:confirm_trade:{pending.id}" - ), - InlineKeyboardButton( - "Reject", callback_data=f"agent:reject_trade:{pending.id}" - ), - ] - ] - ) - # Naming the asker is not decoration: one chat can talk to several - # agents on several servers, and "Approve this action?" on its own does - # not say whose action, or where it lands. - asked_by = f"\nRequested by {pending.origin}\n" if pending.origin else "" - await self._bot.send_message( - chat_id=self._chat_id, - text=( - f"Trade Confirmation\n{asked_by}\n" - f"{pending.summary}\n\nApprove this action?" - ), - reply_markup=keyboard, - ) - - async def permission_callback( bot: Bot, chat_id: int, diff --git a/handlers/agents/custom_models.py b/handlers/agents/custom_models.py index b8f212d35..bf30899b0 100644 --- a/handlers/agents/custom_models.py +++ b/handlers/agents/custom_models.py @@ -1,344 +1,11 @@ -"""Custom OpenAI-compatible provider model fetcher. +"""Moved to :mod:`condor.llm.custom_models` (ARCH-190). -Hits GET {base_url}/models with an optional Bearer key — the standard model -list endpoint every OpenAI-compatible API exposes (Venice AI, Together, -Fireworks, vLLM, TGI, ...). A successful fetch doubles as validation of both -the URL and the API key during the custom-endpoint setup flow. +This alias keeps ``handlers.agents.custom_models`` imports (and monkeypatches +on the module's attributes) pointing at the one real module object. """ -from __future__ import annotations +import sys -import asyncio -import hashlib -import ipaddress -import logging -import os -import re -import socket -from urllib.parse import urlparse +from condor.llm import custom_models as _impl -import aiohttp - -log = logging.getLogger(__name__) - -# Per-candidate budget. Candidates are probed concurrently, so this is also -# roughly the worst case for the whole fetch — the user is staring at a -# "validating..." message the entire time. -FETCH_TIMEOUT_SECONDS = 8 - -# Hosts that hand out cloud instance credentials to anything that can issue an -# HTTP GET. Always refused: no legitimate model endpoint lives here. -BLOCKED_HOSTNAMES = frozenset( - { - "metadata.google.internal", - "metadata.goog", - "instance-data", - } -) - -# Opt-in for multi-tenant deployments: also refuse loopback and RFC1918 -# targets. Off by default because pointing at a local Ollama/vLLM/LM Studio -# server is the single most common use of this feature. -_BLOCK_PRIVATE_ENV = "CUSTOM_LLM_BLOCK_PRIVATE_URLS" - -# /models entries whose declared type isn't one of these are not chat models. -# Providers disagree on the field name (`type` on Venice/Together, `object` on -# stock OpenAI) and on the vocabulary, so accept every spelling of "text in, -# text out" we've seen and treat other declared values as non-chat. -CHAT_MODEL_TYPES = frozenset({"text", "chat", "language", "llm"}) - -# Values that technically occupy the type field but say nothing about -# modality — stock OpenAI stamps object="model" on embeddings, Whisper and -# DALL·E alike. Treat these as "not declared" and fall through to the id. -UNINFORMATIVE_MODEL_TYPES = frozenset({"model", "list", "object"}) - -# Fallback for providers that return no type at all: drop ids that are -# unmistakably a non-chat modality. -_NON_CHAT_MARKERS = ( - "embed", - "rerank", - "whisper", - "tts", - "text-to-speech", - "speech", - "voice", - "moderation", - "guard", - "upscal", - "stable-diffusion", - "flux", - "dall-e", - "sdxl", - "image", - "vision-encoder", - "clip", - "ocr", -) - - -# urlparse is happy to call "not a url at all" a netloc, so the hostname needs -# its own shape check: dotted labels, or an IPv6 literal in brackets. -_HOSTNAME_RE = re.compile( - r"^(?:[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*\.?|\[[0-9A-Fa-f:.]+\])$" -) - - -class CustomProviderError(Exception): - """Raised when the endpoint can't be validated. Message is user-facing.""" - - -def _block_private() -> bool: - return os.environ.get(_BLOCK_PRIVATE_ENV, "").strip().lower() in ( - "1", - "true", - "yes", - "on", - ) - - -def _reject_if_blocked(host: str) -> None: - """Raise ValueError when `host` points somewhere we refuse to fetch from. - - Link-local addresses (the 169.254.169.254 cloud metadata range) are always - refused. Loopback and private ranges are allowed by default so local model - servers work, and can be refused too by setting - CUSTOM_LLM_BLOCK_PRIVATE_URLS=true on shared/multi-tenant deployments. - - Only IP literals are checked unless the strict flag is set — resolving - every hostname would add latency and would still be defeatable by DNS - rebinding, so this is a guard against casual probing, not a security - boundary. - """ - lowered = host.lower().rstrip(".") - if lowered in BLOCKED_HOSTNAMES: - raise ValueError(f"'{host}' is not an allowed endpoint host.") - - strict = _block_private() - addresses: list[str] = [] - try: - ipaddress.ip_address(lowered.strip("[]")) - addresses = [lowered.strip("[]")] - except ValueError: - if strict: - try: - addresses = [info[4][0] for info in socket.getaddrinfo(lowered, None)] - except OSError: - # Unresolvable — let the actual request produce the error - return - - for raw in addresses: - try: - ip = ipaddress.ip_address(raw) - except ValueError: - continue - if ip.is_link_local: - raise ValueError( - f"'{host}' resolves to a link-local address, which is blocked " - "(cloud instance metadata lives there)." - ) - if strict and (ip.is_loopback or ip.is_private or ip.is_reserved): - raise ValueError( - f"'{host}' resolves to a private address and " - f"{_BLOCK_PRIVATE_ENV} is enabled." - ) - - -def normalize_base_url(raw: str) -> str: - """Coerce user input into a usable OpenAI-compatible base URL. - - Adds https:// when no scheme is given and strips trailing slashes and - endpoint suffixes users commonly paste (/chat/completions, /models). - Raises ValueError for input that can't be a URL or that targets a - blocked host. - """ - url = raw.strip() - if not url: - raise ValueError("URL is empty.") - if "://" not in url: - url = f"https://{url}" - url = url.rstrip("/") - - try: - parsed = urlparse(url) - hostname = parsed.hostname - except ValueError: - raise ValueError(f"'{raw.strip()}' is not a valid http(s) URL.") from None - if ( - parsed.scheme not in ("http", "https") - or not parsed.netloc - or not hostname - or not _HOSTNAME_RE.match(hostname) - ): - raise ValueError(f"'{raw.strip()}' is not a valid http(s) URL.") - _reject_if_blocked(hostname) - - for suffix in ("/chat/completions", "/completions", "/models"): - if url.endswith(suffix): - url = url[: -len(suffix)] - break - return url.rstrip("/") - - -def _is_chat_model(entry: dict) -> bool: - """Best-effort guess at whether a /models entry can serve chat completions. - - Providers return embeddings, rerankers, TTS and image models from the same - endpoint. Picking one of those succeeds here and then fails at prompt time - with an opaque upstream error, so filter them out while we still have the - metadata to do it. - """ - declared = entry.get("type") or entry.get("object") - if isinstance(declared, str) and declared.strip(): - normalized = declared.strip().lower() - if normalized not in UNINFORMATIVE_MODEL_TYPES: - return normalized in CHAT_MODEL_TYPES - - capabilities = entry.get("capabilities") - if isinstance(capabilities, dict): - for flag in ("chat", "chat_completions", "supportsChatCompletions"): - if flag in capabilities: - return bool(capabilities[flag]) - - model_id = entry.get("id", "").lower() - return not any(marker in model_id for marker in _NON_CHAT_MARKERS) - - -def _extract_model_ids(payload: object, chat_only: bool = True) -> list[str]: - """Pull model ids from an OpenAI-format /models response. - - Expects {"data": [{"id": "..."}, ...]}. Returns sorted, deduped ids; - empty list when the payload doesn't match the expected shape. With - ``chat_only`` (the default) non-chat modalities are dropped. - """ - if not isinstance(payload, dict): - return [] - data = payload.get("data") - if not isinstance(data, list): - return [] - ids = { - entry["id"].strip() - for entry in data - if isinstance(entry, dict) - and isinstance(entry.get("id"), str) - and entry["id"].strip() - and (not chat_only or _is_chat_model(entry)) - } - return sorted(ids, key=str.lower) - - -async def _probe( - session: aiohttp.ClientSession, candidate: str, headers: dict -) -> tuple[list[str], CustomProviderError | None]: - """Fetch one candidate's model list. Returns (models, error).""" - models_url = f"{candidate}/models" - try: - async with session.get(models_url, headers=headers) as resp: - if resp.status in (401, 403): - return [], CustomProviderError( - f"the provider at {candidate} rejected the API key " - f"(HTTP {resp.status}). Check the key and try again." - ) - if resp.status == 404: - return [], CustomProviderError( - f"no /models endpoint at {candidate} (HTTP 404). " - "Check the base URL — it usually ends in /v1." - ) - if resp.status != 200: - return [], CustomProviderError( - f"the provider at {candidate} returned HTTP {resp.status}." - ) - payload = await resp.json(content_type=None) - except asyncio.CancelledError: - raise - except Exception as e: - return [], CustomProviderError( - f"could not reach {models_url} ({type(e).__name__}). " - "Check the URL and that the server is up." - ) - - model_ids = _extract_model_ids(payload) - if not model_ids: - # An endpoint that answered but listed only embeddings/image models is - # a different failure from one that isn't OpenAI-shaped at all. - if _extract_model_ids(payload, chat_only=False): - return [], CustomProviderError( - f"{models_url} returned models, but none of them are chat " - "models (only embeddings/image/audio)." - ) - return [], CustomProviderError( - f"{models_url} responded but did not return an " - "OpenAI-compatible model list." - ) - return model_ids, None - - -async def fetch_models(base_url: str, api_key: str = "") -> tuple[str, list[str]]: - """Validate an OpenAI-compatible endpoint and return its chat model list. - - Probes {base_url}/models and {base_url}/v1/models concurrently — the second - covers users who pasted the host root (e.g. https://api.venice.ai instead - of https://api.venice.ai/api/v1) — and prefers the first candidate that - works. Returns (resolved_base_url, model_ids). - - Raises CustomProviderError with a user-facing message on any failure — - unreachable host, rejected key, or a non-OpenAI-shaped response. - """ - headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} - candidates = [base_url] - if not base_url.endswith("/v1"): - candidates.append(f"{base_url}/v1") - - timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_SECONDS) - async with aiohttp.ClientSession(timeout=timeout) as session: - results = await asyncio.gather( - *(_probe(session, candidate, headers) for candidate in candidates) - ) - - # Candidate order is preference order: the URL the user actually typed wins - for candidate, (model_ids, _) in zip(candidates, results): - if model_ids: - log.info( - "Custom provider validated: %s (%d chat models)", - candidate, - len(model_ids), - ) - return candidate, model_ids - - for _, error in results: - if error: - raise error - raise CustomProviderError("no reachable endpoint found.") - - -def model_token(model_id: str) -> str: - """Short stable token for a model id, for use in callback_data. - - Telegram caps callback_data at 64 bytes and model ids are unbounded. A - content-derived token also survives the model list being refetched between - render and tap, which a bare list index does not. - """ - return hashlib.blake2s(model_id.encode("utf-8"), digest_size=4).hexdigest() - - -def find_by_token(model_ids: list[str], token: str) -> str | None: - """Resolve a token produced by :func:`model_token` back to its model id.""" - for model_id in model_ids: - if model_token(model_id) == token: - return model_id - return None - - -def format_model_label(model_id: str, limit: int = 38) -> str: - """Short label for inline keyboard buttons. - - Truncates through the middle: for ids like - ``meta-llama/Llama-3.3-70B-Instruct-Turbo-Free`` the tail is what - distinguishes one variant from another, so cutting it off is exactly the - wrong end to lose. - """ - if len(model_id) <= limit: - return model_id - keep = limit - 1 # room for the ellipsis - head = (keep + 1) // 2 - tail = keep - head - return f"{model_id[:head]}…{model_id[-tail:]}" +sys.modules[__name__] = _impl diff --git a/handlers/agents/openrouter_models.py b/handlers/agents/openrouter_models.py index 8a7b17b58..b4ba909a8 100644 --- a/handlers/agents/openrouter_models.py +++ b/handlers/agents/openrouter_models.py @@ -1,119 +1,11 @@ -"""OpenRouter model catalog fetcher. +"""Moved to :mod:`condor.llm.openrouter_models` (ARCH-190). -Hits GET https://openrouter.ai/api/v1/models (public, unauthenticated) and -filters to entries that advertise `tools` in their supported_parameters. -Condor's whole architecture depends on tool-calling, so models without it -are hidden from the picker. - -Results are cached for an hour so paginating the picker doesn't refetch. +This alias keeps ``handlers.agents.openrouter_models`` imports (and +monkeypatches on the module's attributes) pointing at the one real module. """ -from __future__ import annotations - -import logging -import time -from dataclasses import dataclass - -import aiohttp - -log = logging.getLogger(__name__) - -OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" -CACHE_TTL_SECONDS = 3600 - - -@dataclass(frozen=True) -class OpenRouterModel: - slug: str # e.g. "anthropic/claude-sonnet-4-5" - name: str # human-friendly name from the API - context_length: int # tokens - prompt_price: float # USD per 1M input tokens, 0 if free - completion_price: float - - -_cache: tuple[float, list[OpenRouterModel]] | None = None - - -def _parse_price(value: object) -> float: - """OpenRouter returns prices as strings in USD per token. Convert to per-1M.""" - try: - return float(value) * 1_000_000 # type: ignore[arg-type] - except (TypeError, ValueError): - return 0.0 - - -def _model_supports_tools(entry: dict) -> bool: - params = entry.get("supported_parameters") or [] - return "tools" in params - - -async def fetch_models(force_refresh: bool = False) -> list[OpenRouterModel]: - """Return the list of OpenRouter models that support tool-calling. - - Sorted by provider then model id for predictable pagination. - """ - global _cache - - if not force_refresh and _cache is not None: - cached_at, models = _cache - if time.monotonic() - cached_at < CACHE_TTL_SECONDS: - return models - - timeout = aiohttp.ClientTimeout(total=10) - try: - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.get(OPENROUTER_MODELS_URL) as resp: - resp.raise_for_status() - payload = await resp.json() - except Exception as e: - log.warning("Failed to fetch OpenRouter models: %s", e) - if _cache is not None: - return _cache[1] # serve stale cache on failure - return [] - - raw = payload.get("data") or [] - models: list[OpenRouterModel] = [] - for entry in raw: - if not isinstance(entry, dict): - continue - slug = entry.get("id") - if not isinstance(slug, str) or not slug or slug.startswith("~"): - continue # skip canonical aliases like "~anthropic/claude-haiku-latest" - if not _model_supports_tools(entry): - continue - pricing = entry.get("pricing") or {} - models.append( - OpenRouterModel( - slug=slug, - name=str(entry.get("name") or slug), - context_length=int(entry.get("context_length") or 0), - prompt_price=_parse_price(pricing.get("prompt")), - completion_price=_parse_price(pricing.get("completion")), - ) - ) - - models.sort(key=lambda m: m.slug.lower()) - _cache = (time.monotonic(), models) - log.info("Fetched %d OpenRouter models with tool support", len(models)) - return models - - -def format_button_label(model: OpenRouterModel) -> str: - """Short label for inline keyboard buttons. Telegram max ~30 chars looks good.""" - label = model.name or model.slug - if len(label) > 38: - label = label[:35] + "..." - return label +import sys +from condor.llm import openrouter_models as _impl -def find_model_by_slug( - models: list[OpenRouterModel], slug: str -) -> OpenRouterModel | None: - """Case-insensitive exact-match lookup by slug.""" - target = slug.strip().lower() - if not target: - return None - for m in models: - if m.slug.lower() == target: - return m - return None +sys.modules[__name__] = _impl diff --git a/handlers/agents/readiness.py b/handlers/agents/readiness.py index 340177322..5852d7cdd 100644 --- a/handlers/agents/readiness.py +++ b/handlers/agents/readiness.py @@ -1,298 +1,11 @@ -"""One answer to "can Condor actually run this model right now?". +"""Moved to :mod:`condor.llm.readiness` (ARCH-190). -Every surface that offers a model — the ``get_available_models`` MCP tool behind -the agent_builder skill, and the setup wizard (``condor/setup_llm.py``) — asks -this module, so they can never disagree about what a machine can run. The probes -source their commands from :data:`condor.acp.ACP_COMMANDS` and their base URLs -from :data:`condor.acp.pydantic_ai_client.DEFAULT_BASE_URLS`, so adding a -provider there lights it up here for free. - -Three states, never two: a CLI bridge that is installed but whose login cannot be -proven is ``UNVERIFIED``, not ready and not missing. The login markers below are -undocumented paths that upstream CLIs may move, so they only ever downgrade a row -from *ready* to *unverified* — never to *missing*, and never enough to refuse a -pick. - -Read-only and side-effect-free. Never returns a key value, and an unreachable -local server is a normal state (the user simply isn't running it), reported as -``MISSING`` with the command that starts it, never raised. +This alias keeps ``handlers.agents.readiness`` imports (and monkeypatches on +the module's attributes) pointing at the one real module object. """ -from __future__ import annotations - -import os -import shutil -import subprocess import sys -from dataclasses import dataclass, field -from pathlib import Path -from typing import Iterable - -import httpx - -from condor.acp import ACP_COMMANDS -from condor.acp.pydantic_ai_client import DEFAULT_BASE_URLS - -READY, UNVERIFIED, MISSING = "ready", "unverified", "missing" - -# Cloud providers Condor can address as ":", mapped to the env -# var holding each key. Presence of the key = the provider is usable. Sourced -# here (not hardcoded per surface) so the set stays in one place. -CLOUD_KEY_ENVS = { - "openrouter": "OPENROUTER_API_KEY", - "openai": "OPENAI_API_KEY", - "anthropic": "ANTHROPIC_API_KEY", - "groq": "GROQ_API_KEY", - "google": "GOOGLE_API_KEY", -} - -# Local, keyless OpenAI-compatible servers. Base URLs come from the resolver's -# own defaults (DEFAULT_BASE_URLS) so they can never drift from what actually -# runs the model at request time. -LOCAL_PREFIXES = ("ollama", "lmstudio") - -# What to run once, interactively, to log a bridge in. Not run from here: the -# wizard reports and instructs, it never drives someone else's OAuth flow. -_LOGIN_COMMANDS = { - "claude-code": "claude", - "claude-acp": "claude", - "gemini": "npx @google/gemini-cli", - "copilot": "npx @github/copilot", - "codex": "codex", -} - -# Bridges launched as a bare command still come from npm; name the package so a -# missing one is actionable instead of just absent. -_BARE_INSTALL = { - "claude-agent-acp": "npm install -g @agentclientprotocol/claude-agent-acp", -} - -_START_COMMANDS = { - "ollama": "start it with `ollama serve`", - "lmstudio": "start LM Studio and enable its local server", -} - - -@dataclass(frozen=True) -class Readiness: - """What a menu row needs: a state, a sentence, and any models behind it.""" - - state: str # READY | UNVERIFIED | MISSING - detail: str # human sentence; when not READY it names the fix - models: list[str] = field(default_factory=list) # local server models; [] elsewhere - - @property - def usable(self) -> bool: - """False only for MISSING — an unproven login must never block a pick.""" - return self.state != MISSING - - -def npx_packages_installed() -> set[str]: - """npm packages resolvable WITHOUT a network install: local, global, npx cache. - - Mirrors how npx resolves a package before falling back to downloading it. - ``shutil.which("npx")`` says nothing about the package — npx exists whenever - Node does — so probing the launcher would mark every ``npx``-run bridge - available on any machine with Node. - """ - roots = [Path.cwd() / "node_modules"] - npm = shutil.which("npm") - if npm: - try: - proc = subprocess.run( - [npm, "root", "-g"], capture_output=True, text=True, timeout=10 - ) - if proc.returncode == 0 and proc.stdout.strip(): - roots.append(Path(proc.stdout.strip())) - except (OSError, subprocess.SubprocessError): - pass # npm unusable = nothing installed via it; local + cache still count - roots.extend((Path.home() / ".npm" / "_npx").glob("*/node_modules")) - - installed: set[str] = set() - for root in roots: - if not root.is_dir(): - continue - for entry in root.iterdir(): - if entry.name.startswith("@"): - installed.update(f"{entry.name}/{sub.name}" for sub in entry.iterdir()) - else: - installed.add(entry.name) - return installed - - -def _keychain_has(service: str) -> bool | None: - """macOS keychain lookup: True found, False absent, None cannot tell.""" - security = shutil.which("security") - if not security: - return None - try: - proc = subprocess.run( - [security, "find-generic-password", "-s", service], - capture_output=True, - text=True, - timeout=5, - ) - except (OSError, subprocess.SubprocessError): - return None - return proc.returncode == 0 - - -def acp_login_state(base: str) -> bool | None: - """Has this bridge's interactive login already happened? - - ``True`` a credential exists, ``False`` none was found where that CLI keeps - one, ``None`` this bridge has no marker we can read (GitHub Copilot) or the - machine could not be asked. Heuristic by construction: a credentials file - proves a login happened, not that the token is still live, and its absence - proves nothing on a machine using another auth path — which is why callers - treat False and None alike as *unverified*, never as *not installed*. - """ - home = Path.home() - if base in ("claude-code", "claude-acp"): - if (home / ".claude" / ".credentials.json").exists(): - return True - if sys.platform == "darwin": # Claude Code stores it in the login keychain - return _keychain_has("Claude Code-credentials") - return False - if base == "gemini": - if (home / ".gemini" / "oauth_creds.json").exists(): - return True - return bool(os.environ.get("GEMINI_API_KEY")) or False - if base == "codex": - return (home / ".codex" / "auth.json").exists() - return None # copilot and anything new: no documented on-disk marker - - -def install_command(cmd: str) -> str: - """How to install the CLI behind an ACP command line.""" - parts = cmd.split() - if len(parts) > 1 and parts[0] == "npx": - return f"npm install -g {parts[1]}" - if parts and parts[0] in _BARE_INSTALL: - return _BARE_INSTALL[parts[0]] - return f"install `{cmd.split()[0] if parts else cmd}` and put it on PATH" - - -def acp_bridges() -> list[dict]: - """ACP CLI bridges: whether each is installed, and whether it is logged in. - - A bridge launched as ``npx …`` is available only if the PACKAGE is - already installed; a bare command is available if it is on PATH. ``available`` - means "launchable" — a bridge additionally needs its own interactive login, - reported separately as ``logged_in`` (``None`` = no marker to read). - """ - npx_packages = npx_packages_installed() - out: list[dict] = [] - for base, cmd in ACP_COMMANDS.items(): - parts = cmd.split() - if not parts: - available = False - elif parts[0] == "npx": - available = parts[1] in npx_packages if len(parts) > 1 else False - else: - available = shutil.which(parts[0]) is not None - out.append( - { - "agent_key": base, - "command": cmd, - "available": available, - "logged_in": acp_login_state(base) if available else None, - } - ) - return out - - -async def local_servers() -> dict: - """Probe local OpenAI-compatible servers and list their loaded models. - - Unreachable servers are reported (``reachable: false``), not raised — the - user just isn't running that server. - """ - result: dict = {} - timeout = httpx.Timeout(connect=1.5, read=3.0, write=1.5, pool=1.5) - async with httpx.AsyncClient(timeout=timeout) as client: - for prefix in LOCAL_PREFIXES: - base = DEFAULT_BASE_URLS.get(prefix) - entry = {"base_url": base, "reachable": False, "models": []} - if base: - try: - resp = await client.get(f"{base.rstrip('/')}/models") - if resp.status_code == 200: - data = resp.json().get("data", []) - entry["reachable"] = True - entry["models"] = sorted( - m["id"] for m in data if isinstance(m, dict) and m.get("id") - ) - except Exception: - pass # not running = normal, leave reachable=False - result[prefix] = entry - return result - - -def _bridge_readiness(entry: dict) -> Readiness: - base, cmd = entry["agent_key"], entry["command"] - if not entry["available"]: - return Readiness(MISSING, f"not installed — {install_command(cmd)}") - login = _LOGIN_COMMANDS.get(base, cmd) - if entry.get("logged_in") is True: - return Readiness(READY, "installed and logged in") - if entry.get("logged_in") is False: - return Readiness(UNVERIFIED, f"installed; run `{login}` once and log in") - return Readiness( - UNVERIFIED, - f"installed; login could not be verified — if your first chat " - f"fails, run `{login}` once and log in", - ) - - -def _local_readiness(prefix: str, entry: dict) -> Readiness: - if not entry["reachable"]: - hint = _START_COMMANDS.get(prefix, "start the server") - return Readiness(MISSING, f"not reachable at {entry['base_url']} — {hint}") - models = list(entry["models"]) - return Readiness(READY, f"{len(models)} model(s) available", models) - - -def _key_readiness(base: str, env: dict[str, str]) -> Readiness: - var = CLOUD_KEY_ENVS[base] - if env.get(var) or os.environ.get(var): - return Readiness(READY, f"{var} is set") - if base == "openrouter": - return Readiness(MISSING, "needs an API key (you can enter one here)") - return Readiness(MISSING, f"needs {var} in .env") - - -async def probe_all( - bases: Iterable[str], env: dict[str, str] | None = None -) -> dict[str, Readiness]: - """Readiness for several provider bases, probing each source exactly once. - - ``env`` is passed in rather than read from the process so the setup wizard - can probe against a ``.env`` the bot has not loaded yet; ``os.environ`` is - still consulted as a fallback. - """ - env = env or {} - wanted = list(dict.fromkeys(bases)) - bridges: dict[str, dict] = {} - if any(b in ACP_COMMANDS for b in wanted): - bridges = {e["agent_key"]: e for e in acp_bridges()} - servers: dict = {} - if any(b in LOCAL_PREFIXES for b in wanted): - servers = await local_servers() - - out: dict[str, Readiness] = {} - for base in wanted: - if base in bridges: - out[base] = _bridge_readiness(bridges[base]) - elif base in servers: - out[base] = _local_readiness(base, servers[base]) - elif base in CLOUD_KEY_ENVS: - out[base] = _key_readiness(base, env) - else: - out[base] = Readiness(UNVERIFIED, "cannot be checked from here") - return out +from condor.llm import readiness as _impl -async def probe(base: str, env: dict[str, str] | None = None) -> Readiness: - """Readiness for a single provider base (see :func:`probe_all`).""" - return (await probe_all([base], env))[base] +sys.modules[__name__] = _impl diff --git a/handlers/bots/_shared.py b/handlers/bots/_shared.py index c21a89985..8c4777279 100644 --- a/handlers/bots/_shared.py +++ b/handlers/bots/_shared.py @@ -11,7 +11,6 @@ """ import logging -import re from typing import Any, Dict, List, Optional, Tuple from condor.cache import DEFAULT_CACHE_TTL @@ -189,44 +188,13 @@ def set_controller_config(context, config: Dict[str, Any]) -> None: context.user_data["controller_config_params"] = config -# Matches stringified enum values like "PositionMode.ONEWAY", "OrderType.LIMIT", -# "TradeType.BUY" or the colon variant "PositionMode:ONEWAY". The class name is -# PascalCase and the member starts with an uppercase letter, which avoids matching -# legitimate values such as trading times ("09:30") or token symbols ("USDC.e"). -_ENUM_STR_RE = re.compile(r"^[A-Z][A-Za-z0-9_]*[.:][A-Z][A-Za-z0-9_]*$") - - -def normalize_enum_value(value: Any) -> Any: - """Strip the enum-class prefix from stringified enum values. - - Controller config templates expose enum defaults as strings like - "PositionMode.ONEWAY". Saving them verbatim breaks downstream controller - validation, which expects the plain member name ("ONEWAY"). This recurses - into dicts and lists so nested config values are normalized too. - """ - if isinstance(value, str) and _ENUM_STR_RE.match(value): - return re.split(r"[.:]", value, maxsplit=1)[1] - if isinstance(value, dict): - return {k: normalize_enum_value(v) for k, v in value.items()} - if isinstance(value, list): - return [normalize_enum_value(v) for v in value] - return value - - -def clean_config_for_save(config: Dict[str, Any]) -> Dict[str, Any]: - """Strip internal fields and normalize stringified enum values before saving. - - The backend adds _config_name to YAML files, but Pydantic models with - extra='forbid' reject it on reload. Strip any underscore-prefixed keys - that aren't part of the controller config schema. - - Enum fields (e.g. position_mode, side, take_profit_order_type) can arrive as - prefixed strings like "PositionMode.ONEWAY"; normalize them to the plain - member name so deploy/backtest validation accepts them. - """ - return { - k: normalize_enum_value(v) for k, v in config.items() if not k.startswith("_") - } +# Moved to condor/controller_configs.py (ARCH-190) so the web dashboard can use +# them without importing the handlers package; re-imported here for Telegram +# callers. +from condor.controller_configs import ( # noqa: E402,F401 + clean_config_for_save, + normalize_enum_value, +) def init_new_controller_config( diff --git a/handlers/dex/geckoterminal.py b/handlers/dex/geckoterminal.py index 7255edeab..7e0a389b3 100644 --- a/handlers/dex/geckoterminal.py +++ b/handlers/dex/geckoterminal.py @@ -25,6 +25,7 @@ gecko_request, get_connector_for_dex, ) +from config_manager import get_client from utils.telegram_formatters import escape_markdown_v2 from ._shared import cached_call @@ -2303,9 +2304,7 @@ async def show_gecko_liquidity( bins, pool_info, error = await fetch_liquidity_bins( pool_address=address, connector=connector, - user_data=context.user_data, - chat_id=chat_id, - context=context, + client=await get_client(chat_id, context=context), ) if error or not bins: @@ -2454,9 +2453,7 @@ async def show_gecko_combined( bins, pool_info, _ = await fetch_liquidity_bins( pool_address=address, connector=connector, - user_data=context.user_data, - chat_id=chat_id, - context=context, + client=await get_client(chat_id, context=context), ) if not ohlcv_data and not bins: diff --git a/handlers/dex/pools.py b/handlers/dex/pools.py index b66e94001..0c7b7c336 100644 --- a/handlers/dex/pools.py +++ b/handlers/dex/pools.py @@ -1139,7 +1139,6 @@ async def fetch_ohlcv_task(): network=gecko_network, timeframe=timeframe, currency="token", - user_data=context.user_data, ) if error: logger.warning(f"Failed to fetch OHLCV: {error}") @@ -2026,7 +2025,6 @@ async def handle_pool_ohlcv( network=network, timeframe=timeframe, currency="token", - user_data=context.user_data, ) if error or not ohlcv_data: @@ -2238,7 +2236,6 @@ async def handle_pool_combined_chart( network=network, timeframe=timeframe, currency="token", - user_data=context.user_data, ) # Get bins from cached pool_info or fetch @@ -2248,9 +2245,7 @@ async def handle_pool_combined_chart( bins, _, _ = await fetch_liquidity_bins( pool_address=pool_address, connector=connector, - user_data=context.user_data, - chat_id=chat_id, - context=context, + client=await get_client(chat_id, context=context), ) if not ohlcv_data and not bins: @@ -2930,7 +2925,6 @@ async def fetch_ohlcv_task(): network=gecko_network, timeframe=timeframe, currency="token", - user_data=context.user_data, ) return data or [] except Exception as e: @@ -4365,7 +4359,6 @@ async def show_add_position_menu( network=gecko_network, timeframe=timeframe, currency="token", - user_data=context.user_data, ) if ohlcv_error: logger.warning(f"OHLCV fetch error: {ohlcv_error}") diff --git a/handlers/routines/__init__.py b/handlers/routines/__init__.py index 41609f31b..5e441e6de 100644 --- a/handlers/routines/__init__.py +++ b/handlers/routines/__init__.py @@ -320,7 +320,13 @@ async def _execute_routine( try: config = routine.config_class(**config_dict) - raw_result = await routine.run_fn(config, context) + # Reports belong to the user whose user_data the run executes against + # (the starter), falling back to the chat — the same identity the hooks + # record (SEC-152) — so the web can authorize reads by it (SEC-196). + with condor.reports.attribute_owner( + owner_id if owner_id is not None else chat_id + ): + raw_result = await routine.run_fn(config, context) rich_result = normalize_result(raw_result) result_text = rich_result.text[:500] if rich_result.text else "Completed" except Exception as e: @@ -420,7 +426,10 @@ def user_data(self): try: config = routine.config_class(**config_dict) logger.info(f"Starting continuous routine {routine_name}[{instance_id}]") - result = await routine.run_fn(config, context) + # Same owner stamping as one-shot runs (SEC-196): reports this loop + # saves belong to the user the instance runs for. + with condor.reports.attribute_owner(owner_key): + result = await routine.run_fn(config, context) logger.info(f"Continuous routine {routine_name}[{instance_id}] ended: {result}") except asyncio.CancelledError: logger.info(f"Continuous routine {routine_name}[{instance_id}] cancelled") diff --git a/main.py b/main.py index db1b7c4c4..22d6a609e 100644 --- a/main.py +++ b/main.py @@ -23,7 +23,16 @@ from condor.telemetry import taps as telemetry_taps from handlers import cancel_command, clear_all_input_states from utils.auth import restricted -from utils.config import TELEGRAM_TOKEN, WEB_PORT, WEB_URL +from utils.config import ( + LOCAL_MODE, + TELEGRAM_TOKEN, + WEB_HOST, + WEB_PORT, + WEB_URL, + ConfigError, + check_local_user, + check_startup_config, +) # Enable logging logging.basicConfig( @@ -516,6 +525,8 @@ async def _notify_interrupted_runs(bot, report) -> None: One summary per chat rather than a message per run: a crash with several live loops would otherwise spam the user at the worst possible moment. """ + from condor.notifications import NotifyBot + by_chat: dict[int, list] = {} for run in report.interrupted: status = None @@ -534,10 +545,42 @@ async def _notify_interrupted_runs(bot, report) -> None: for run in runs: suffix = " — restarted" if run.restarted else "" lines.append(f"• {run.label} (last tick {run.last_tick}){suffix}") + text = "\n".join(lines) try: - await bot.send_message(chat_id=chat_id, text="\n".join(lines)) + await bot.send_message(chat_id=chat_id, text=text) except Exception: logger.warning("Could not notify chat %s about interrupted runs", chat_id) + # And on the bell (FEAT-048). A private chat id is the owner's user id; + # ``record`` ignores anything that is not one, so a group summary is + # simply not filed anywhere. Skipped when the sender above *is* the bell + # (local mode, FEAT-049): it already filed exactly this text. + if isinstance(bot, NotifyBot): + continue + try: + from condor.notifications import record, user_for_chat + + owner = user_for_chat(chat_id) + if owner: + await record(owner, text, kind="system") + except Exception: + logger.debug("Could not record interrupted-run notice", exc_info=True) + + +def _outbound_bot(application: Application): + """The object this process sends user-facing messages through. + + Telegram mode: the real PTB bot, exactly as before. Local mode: there is no + Telegram behind the placeholder token, so outbound messages go to the + dashboard bell instead (:class:`condor.notifications.NotifyBot`, FEAT-048). + Everything downstream — the routine store, the session health monitor, the + interrupted-run summaries — keeps calling ``send_message`` and neither knows + nor cares which surface it reached. + """ + if not LOCAL_MODE: + return application.bot + from condor.notifications import NotifyBot + + return NotifyBot() async def startup(application: Application) -> None: @@ -560,8 +603,16 @@ async def startup(application: Application) -> None: asyncio.get_event_loop().run_in_executor(None, _get_model, DEFAULT_MODEL) - # Register command menus (public + admin overlay) - await register_bot_commands(application) + # Whatever this process pushes at users from here on. In local mode there is + # no Telegram to push to, so it is the dashboard bell (FEAT-048) instead — + # which is what keeps the documented "context.bot is never None" contract + # true for routines with no bot behind them. + outbound_bot = _outbound_bot(application) + + # Register command menus (public + admin overlay). Pure Telegram: there is + # no command menu to publish when nothing polls. + if not LOCAL_MODE: + await register_bot_commands(application) # Restore scheduled routine jobs from persistence from handlers.routines import restore_scheduled_jobs @@ -571,7 +622,7 @@ async def startup(application: Application) -> None: # Inject Telegram bot into routine store so web-triggered routines can send messages from condor.routine_store import get_routine_store - get_routine_store().set_bot(application.bot) + get_routine_store().set_bot(outbound_bot) # Start ServerDataService (unified server-centric cache) from condor.server_data_service import get_server_data_service @@ -588,7 +639,7 @@ async def startup(application: Application) -> None: from condor.runtime import sessions as runtime_sessions from condor.runtime.confirmations import get_registry - await runtime_sessions.start_health_monitor(application.bot) + await runtime_sessions.start_health_monitor(outbound_bot) # Sweeps expired approvals so a request nobody answers is denied, not leaked. await get_registry().start() @@ -604,7 +655,7 @@ async def startup(application: Application) -> None: report.total, len(report.restarted), ) - await _notify_interrupted_runs(application.bot, report) + await _notify_interrupted_runs(outbound_bot, report) except Exception: logger.exception("Boot reconciliation failed; continuing startup") @@ -790,6 +841,19 @@ async def send_to_all(self, message: str, parse_mode: str = "Markdown"): def main() -> None: """Run the bot.""" + # Refuse to start on a configuration that cannot mean what it says: telegram + # mode (the default) with no token used to surface as an InvalidToken + # traceback from inside PTB, and must never be quietly read as "local mode". + # check_local_user() is the same idea one layer in — local mode logs in with + # no password, so who it logs in as is settled here, at boot, and not as a + # 500 from /auth/local-login with the browser already open. + try: + check_startup_config() + check_local_user() + except ConfigError as exc: + logger.error("%s", exc) + raise SystemExit(1) + # Reap any ACP/MCP subprocess trees orphaned by a prior hard kill (kill -9, # OOM, power loss) before we spawn our own — those bypass teardown(). try: @@ -808,9 +872,12 @@ def main() -> None: # Create the Application with persistence enabled. No post_init/post_shutdown # hooks: they only fire from run_polling/run_webhook, and _run_dual owns the # lifecycle — startup() and teardown() are called there, explicitly. + # In local mode there is no token and nothing polls; the placeholder exists + # only so the Application (and with it job_queue, CallbackContext and the + # handler registry) can be built at all. Nothing ever calls Telegram with it. application = ( Application.builder() - .token(TELEGRAM_TOKEN) + .token(TELEGRAM_TOKEN or "0:local") .persistence(persistence) .concurrent_updates(True) .build() @@ -837,6 +904,25 @@ def main() -> None: exec_restart() # never returns +def _web_server_config(web_app): + """uvicorn's config for the dashboard, isolated so the bind address is testable. + + ``WEB_HOST`` is ``0.0.0.0`` in telegram mode (unchanged) and loopback in + local mode, where the dashboard has no login at all — see + :func:`utils.config.resolve_web_host` for why that is not negotiable by + accident. + """ + import uvicorn + + return uvicorn.Config( + web_app, + host=WEB_HOST, + port=WEB_PORT, + log_level="info", + access_log=False, + ) + + async def _run_dual(application: Application) -> None: """Run the Telegram bot and FastAPI web server concurrently.""" import signal @@ -850,21 +936,24 @@ async def _run_dual(application: Application) -> None: # initialize() and start_polling() — the same slot PTB gives post_init — so # commands, caches and boot reconciliation are settled before the first # update is dispatched. - await application.initialize() - await startup(application) - await application.updater.start_polling(allowed_updates=Update.ALL_TYPES) - await application.start() + # + # Local mode skips that lifecycle rather than faking it (FEAT-049): the + # Application is built but never initialized, so nothing polls and no handler + # can dispatch — the Telegram surface is inert, not mocked. The job queue is + # started directly, which is all scheduled routines, update checks and + # signals actually need. + if LOCAL_MODE: + await startup(application) + await application.job_queue.start() + else: + await application.initialize() + await startup(application) + await application.updater.start_polling(allowed_updates=Update.ALL_TYPES) + await application.start() # Create and start the web server web_app = create_app() - config = uvicorn.Config( - web_app, - host="0.0.0.0", - port=WEB_PORT, - log_level="info", - access_log=False, - ) - server = uvicorn.Server(config) + server = uvicorn.Server(_web_server_config(web_app)) # Start WebSocket manager get_ws_manager().start() @@ -882,21 +971,42 @@ async def _run_dual(application: Application) -> None: get_current_branch(), get_local_commit() ) version = f" ({branch} @ {commit})" if commit else "" - await application.bot.send_message( - chat_id=int(ADMIN_USER_ID), - text=f"Condor is online and ready.{version}", - ) + boot_text = f"Condor is online and ready.{version}" + if not LOCAL_MODE: + await application.bot.send_message( + chat_id=int(ADMIN_USER_ID), + text=boot_text, + ) + # The same notice on the dashboard bell (FEAT-048), so an admin who + # only has the browser open still sees which commit came up. + from condor.notifications import record + + await record(int(ADMIN_USER_ID), boot_text, kind="system") except Exception as e: logger.warning(f"Failed to send startup notification to admin: {e}") # Ask, once, whether this install wants to be counted (FEAT-023). Sent # next to the boot notification because that is the one moment the admin - # is already looking. Until it is answered, nothing is collected. - from condor.telemetry.prompt import maybe_prompt_admin - - await maybe_prompt_admin(application.bot) - - logger.info("Starting Condor: Telegram bot + web dashboard on port %s", WEB_PORT) + # is already looking. Until it is answered, nothing is collected. The + # prompt is a Telegram message with inline buttons, so local mode is + # skipped here and asked in the dashboard instead (the consent card in + # Settings → Privacy). Either way the install is already counted: + # `telemetry.init()` does that at the ping floor, without an answer. + if not LOCAL_MODE: + from condor.telemetry.prompt import maybe_prompt_admin + + await maybe_prompt_admin(application.bot) + + if LOCAL_MODE: + logger.info( + "Starting Condor in local mode (no Telegram): dashboard on http://%s:%s", + WEB_HOST, + WEB_PORT, + ) + else: + logger.info( + "Starting Condor: Telegram bot + web dashboard on port %s", WEB_PORT + ) # Handle shutdown signals shutdown_event = asyncio.Event() @@ -927,12 +1037,22 @@ def _signal_handler(): logger.exception("Web server crashed during shutdown") # Run each step independently so one failure can't skip the rest. # teardown() runs last, mirroring where post_shutdown would have sat. - for name, step in ( - ("updater.stop", application.updater.stop), - ("application.stop", application.stop), - ("application.shutdown", application.shutdown), - ("teardown", partial(teardown, application)), - ): + # Local mode never initialized or started the Application, so the PTB + # stop steps would only raise "not running" — it has a job queue to stop + # and nothing else. + if LOCAL_MODE: + steps = ( + ("job_queue.stop", application.job_queue.stop), + ("teardown", partial(teardown, application)), + ) + else: + steps = ( + ("updater.stop", application.updater.stop), + ("application.stop", application.stop), + ("application.shutdown", application.shutdown), + ("teardown", partial(teardown, application)), + ) + for name, step in steps: try: await step() except Exception: diff --git a/setup-environment.sh b/setup-environment.sh index 47e596b72..f64692a1d 100755 --- a/setup-environment.sh +++ b/setup-environment.sh @@ -68,6 +68,55 @@ escape_env_value() { echo "$value" } +# Set (or replace) KEY=VALUE in .env, creating the file if it does not exist. +set_env_var() { + local key="$1" + local value + value="$(escape_env_value "$2")" + [ -f "$ENV_FILE" ] || touch "$ENV_FILE" + if grep -q "^${key}=" "$ENV_FILE"; then + sed -i.bak "s|^${key}=.*|${key}=${value}|" "$ENV_FILE" + rm -f "$ENV_FILE.bak" + else + echo "${key}=${value}" >> "$ENV_FILE" + fi +} + +# Read one KEY out of .env *without executing the file*. +# +# `source` runs .env as a shell script, so one value containing shell +# metacharacters (an API key with parentheses is enough) is a syntax error that +# aborts the read and silently drops every variable below that line. That is how +# a hand-added CONDOR_MODE=local could become invisible here — and then be +# rewritten to `telegram` by the legacy-install migration below. The variables +# that decide the mode are read with this instead. +read_env_var() { + local key="$1" line + [ -f "$ENV_FILE" ] || return 0 + line="$(grep -E "^[[:space:]]*(export[[:space:]]+)?${key}[[:space:]]*=" "$ENV_FILE" 2>/dev/null | tail -1)" || return 0 + [ -n "$line" ] || return 0 + line="${line#*=}" + line="$(printf '%s' "$line" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')" + case "$line" in + \"*\") line="${line%\"}"; line="${line#\"}" ;; + \'*\') line="${line%\'}"; line="${line#\'}" ;; + esac + printf '%s' "$line" +} + +# What local mode costs, said out loud. It is a dashboard with no login at all, +# so the only thing standing between it and the network is the loopback bind. +local_mode_warning() { + echo "" + msg_warn "Local mode has NO login — whoever reaches the dashboard controls your funds" + echo -e " • It binds ${BOLD}127.0.0.1${RESET} only: reachable from this machine, nowhere else." + echo -e " • Exposing it takes an explicit ${BOLD}WEB_HOST=0.0.0.0${RESET} in .env, which puts full" + echo -e " trading control in reach of anyone who can hit the port. Only do that behind" + echo -e " something that authenticates (Tailscale, an SSH tunnel, a reverse proxy)." + echo -e " • Dashboard: ${BOLD}http://localhost:8088${RESET}" + echo "" +} + # OSC 8 clickable hyperlink (falls back to plain URL) make_link() { local url="$1" @@ -456,25 +505,104 @@ fi echo "" -# ── Step 1: Telegram Configuration ────────────────── +# ── Step 1: Run mode (Telegram or local) ──────────── -echo -e "${BOLD}Step 1: Telegram Configuration${RESET}" +echo -e "${BOLD}Step 1: How will you use Condor?${RESET}" echo "" telegram_configured=false -# Source existing .env if present +# Source existing .env if present. Best-effort only: the rest of the wizard +# reads incidental values (DEPLOY_HUMMINGBOT_API, CONDOR_DEFAULT_AGENT, the +# Tailscale settings) from here, but nothing that decides the *mode* — those are +# re-read below with read_env_var, which cannot be defeated by one unquoted +# value halfway down the file. if [ -f "$ENV_FILE" ]; then set -a source "$ENV_FILE" 2>/dev/null set +a fi -if [ -n "${TELEGRAM_TOKEN:-}" ] && [ -n "${ADMIN_USER_ID:-}" ]; then - msg_ok "Telegram already configured" - telegram_configured=true - load_tailscale_choice +if [ -f "$ENV_FILE" ]; then + CONDOR_MODE="$(read_env_var CONDOR_MODE)" + TELEGRAM_TOKEN="$(read_env_var TELEGRAM_TOKEN)" + ADMIN_USER_ID="$(read_env_var ADMIN_USER_ID)" +fi + +# An install from before local mode existed has a token but no CONDOR_MODE. +# That is telegram — recorded now, so the mode is always explicit on disk and +# is never inferred later from whether a token happens to be there. +if [ -z "${CONDOR_MODE:-}" ] && [ -n "${TELEGRAM_TOKEN:-}" ] && [ -n "${ADMIN_USER_ID:-}" ]; then + CONDOR_MODE="telegram" + set_env_var CONDOR_MODE "telegram" +fi + +# A configured install is *offered the other mode*, never just told it is +# already configured. Switching is the one thing a re-run has to be able to do: +# the boot errors for a broken mode all say "run make setup", and that has to be +# advice that leads somewhere. +choose_mode() { + echo -e " ${BOLD}1) Telegram${RESET} — control Condor from a Telegram bot (recommended)" + echo -e " ${BOLD}2) Local${RESET} — no Telegram; the dashboard runs on this machine, no login" + echo "" + while true; do + prompt_visible "Choose [1/2]" "$1" "condor_mode_choice" + case "${condor_mode_choice:-}" in + 1|2) break ;; + *) msg_warn "Enter 1 (Telegram) or 2 (Local)" ;; + esac + done + echo "" +} + +if [ "${CONDOR_MODE:-}" = "local" ]; then + msg_ok "Currently: Local mode — dashboard only, no Telegram" + echo "" + prompt_visible "Keep local mode? [Y/n]" "Y" "keep_mode" + echo "" + if [[ "${keep_mode:-Y}" =~ ^[Nn]$ ]]; then + choose_mode "1" + else + condor_mode_choice=2 + fi +elif [ -n "${TELEGRAM_TOKEN:-}" ] && [ -n "${ADMIN_USER_ID:-}" ]; then + msg_ok "Currently: Telegram (bot configured, admin ${ADMIN_USER_ID})" + echo "" + prompt_visible "Keep Telegram mode? [Y/n]" "Y" "keep_mode" + echo "" + if [[ "${keep_mode:-Y}" =~ ^[Nn]$ ]]; then + choose_mode "2" + else + telegram_configured=true + load_tailscale_choice + fi else + choose_mode "1" +fi + +if [ "${condor_mode_choice:-}" = "2" ]; then + # Local mode: nothing to ask. No bot, no token — the dashboard logs in as + # ADMIN_USER_ID, the same id config.yml, chat defaults, preferences, memory + # and session keys already key on. + # + # An existing numeric id is KEPT, never overwritten with 1: a Telegram + # install switching to local keeps its own admin, so the dashboard logs in + # as you with every server, preference and default intact. 1 is only the + # answer for an install that never had a Telegram id. + CONDOR_MODE="local" + USE_TAILSCALE=false + if [[ "${ADMIN_USER_ID:-}" =~ ^[1-9][0-9]*$ ]]; then + msg_info "Local mode will log in as user ${ADMIN_USER_ID} (ADMIN_USER_ID in .env)" + else + ADMIN_USER_ID="1" + set_env_var ADMIN_USER_ID "1" + fi + set_env_var CONDOR_MODE "local" + set_env_var WEB_URL "http://localhost:8088" + set_env_var USE_TAILSCALE "false" + msg_ok ".env configured for local mode" + local_mode_warning +elif [ "${condor_mode_choice:-}" = "1" ]; then msg_info "Create a bot: $(make_link 'https://t.me/BotFather')" msg_info "Get your ID: $(make_link 'https://t.me/userinfobot')" echo "" @@ -571,6 +699,11 @@ else } > "$ENV_FILE" fi + # The mode is written explicitly even for the default, so nothing downstream + # has to guess it from the presence of a token. + CONDOR_MODE="telegram" + set_env_var CONDOR_MODE "telegram" + msg_ok ".env created" if [ -n "$SERVER_IP" ]; then msg_ok "Web server configured: http://$SERVER_IP:8088" @@ -1080,12 +1213,18 @@ echo -e " • npm: $(command_exists npm && npm --version 2>/dev/null | echo -e " • typescript: $(command_exists tsc && tsc --version 2>/dev/null || echo 'not installed')" echo "" echo -e " ${BOLD}Configuration:${RESET}" +echo -e " • Mode: ${CONDOR_MODE:-telegram}$([ "${CONDOR_MODE:-telegram}" = "local" ] && echo ' (no Telegram, dashboard only)')" echo -e " • Telegram: $([ -n "${TELEGRAM_TOKEN:-}" ] && echo 'configured' || echo 'not set')" echo -e " • AI model: ${CONDOR_DEFAULT_AGENT:-from agents/condor/AGENT.md}" echo "" echo -e " ${BOLD}Next steps:${RESET}" echo -e " ${BOLD}make install${RESET} Install Python dependencies" echo -e " ${BOLD}make run${RESET} Run Condor locally (dev)" +if [ "${CONDOR_MODE:-telegram}" = "local" ]; then +echo "" +echo -e " Then open ${BOLD}http://localhost:8088${RESET} — you are logged in already." +echo -e " ${DIM}No login, loopback only. See 'Local mode' in the README before exposing it.${RESET}" +fi if [ "${hb_api_deployed:-}" = true ]; then echo "" echo -e " Hummingbot API is running — config at ${BOLD}../hummingbot-api/.env${RESET}" diff --git a/tests/conftest.py b/tests/conftest.py index 57edee04f..82768dfa0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,3 +43,17 @@ def _reset_gecko_throttle(): reset_gecko_throttle() yield + + +@pytest.fixture(autouse=True) +def _isolated_notification_store(tmp_path, monkeypatch): + """Keep the bell's store out of the developer's ``data/`` directory. + + ``condor.notifications.record`` is now reached from several producers + (FEAT-048), so without this any test that finishes a delegation, a routine + or a ``notify`` call appends to the real ``data/notifications.json`` — and a + test run would show up in the running install's notification bell. + """ + from condor import notifications + + monkeypatch.setattr(notifications, "_FILE", tmp_path / "notifications.json") diff --git a/tests/runtime/test_agent_binding.py b/tests/runtime/test_agent_binding.py index 8619269d1..bcec33ed4 100644 --- a/tests/runtime/test_agent_binding.py +++ b/tests/runtime/test_agent_binding.py @@ -64,8 +64,8 @@ def agents_root(tmp_path, monkeypatch): @pytest.fixture def registry(monkeypatch): monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _FakeClient) - monkeypatch.setattr(session_module, "PydanticAIClient", _FakeClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _FakeClient) + monkeypatch.setattr("condor.acp.pydantic_ai_client.PydanticAIClient", _FakeClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "CHAT") _FakeClient.last = None return session_module @@ -101,7 +101,7 @@ def fake_session_servers(user_id, chat_id, user_data=None, **kw): return [{"name": "condor"}] monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", fake_session_servers + "condor.runtime.toolsets.build_mcp_servers_for_session", fake_session_servers ) bound = binding.resolve(_spec(agent_key="claude-code")) @@ -162,7 +162,7 @@ def test_a_missing_default_record_still_starts_the_chat(monkeypatch, tmp_path): """An unreadable agents/condor/AGENT.md degrades; it does not fail closed.""" monkeypatch.setattr(agent_module, "_DATA_ROOT", tmp_path) monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) bound = binding.resolve(_spec(agent_key="claude-code")) @@ -181,7 +181,7 @@ def test_a_named_agent_that_does_not_exist_still_raises(monkeypatch, tmp_path): def test_resolve_agent_sets_memory_scope(agents_root, monkeypatch): """A bound Agent scopes the MCP subprocess to its own store.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [{"name": "condor", "agent_slug": kw.get("agent_slug")}], ) @@ -203,7 +203,7 @@ def test_resolve_agent_sets_memory_scope(agents_root, monkeypatch): def test_agent_key_precedence(agents_root, monkeypatch): """Explicit spec model > Agent's configured model.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) inherited = binding.resolve(_spec(agent_slug="executor_manager")) @@ -228,7 +228,7 @@ def test_unknown_agent_slug(agents_root): def test_tool_allowlist_enforced(agents_root, registry, monkeypatch): """A bound pydantic-ai session is restricted to the Agent's tools.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) info = asyncio.run(runtime.create_session(_spec(agent_slug="executor_manager"))) @@ -246,7 +246,7 @@ def test_tool_allowlist_enforced(agents_root, registry, monkeypatch): def test_bound_session_opens_with_agent_identity(agents_root, registry, monkeypatch): """The Agent's own instructions replace the chat assistant's context.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) asyncio.run( @@ -262,7 +262,7 @@ def test_bound_session_opens_with_agent_identity(agents_root, registry, monkeypa def test_unbound_session_keeps_chat_context(registry, monkeypatch): """Without a binding, the assistant context is untouched.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) asyncio.run( @@ -278,7 +278,7 @@ def test_unbound_session_keeps_chat_context(registry, monkeypatch): def test_switch_destroys_and_respawns(agents_root, registry, monkeypatch): """Rebinding stops the old subprocess exactly once and reuses the key.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) key = SessionKey.telegram(42) @@ -309,7 +309,7 @@ async def scenario(): def test_rebinding_to_same_agent_reuses_session(agents_root, registry, monkeypatch): """An empty agent_key must not look like a model change and respawn.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) async def scenario(): diff --git a/tests/runtime/test_busy_policy.py b/tests/runtime/test_busy_policy.py index 6633642a9..a614a3c89 100644 --- a/tests/runtime/test_busy_policy.py +++ b/tests/runtime/test_busy_policy.py @@ -75,10 +75,10 @@ def registry(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _GatedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _GatedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) return session_module diff --git a/tests/runtime/test_confirmations.py b/tests/runtime/test_confirmations.py index e40b9dac9..da85efde6 100644 --- a/tests/runtime/test_confirmations.py +++ b/tests/runtime/test_confirmations.py @@ -183,7 +183,7 @@ def test_blocked_tool_refused(registry, monkeypatch): the gate still fires the moment something is added to that set. """ monkeypatch.setattr( - "handlers.agents._shared.BLOCKED_TOOLS", {"place_order"}, raising=False + "condor.runtime.danger.BLOCKED_TOOLS", {"place_order"}, raising=False ) channel = _RecordingChannel() cb = build_permission_callback("tg:1", 1, [channel]) diff --git a/tests/runtime/test_conversations.py b/tests/runtime/test_conversations.py index 8dfbc830f..f9ac559f0 100644 --- a/tests/runtime/test_conversations.py +++ b/tests/runtime/test_conversations.py @@ -674,10 +674,10 @@ async def prompt_stream(self, text): @pytest.fixture def registry(monkeypatch, conv_root): monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _ScriptedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _ScriptedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) _ScriptedClient.script = [ TextChunk(text="the answer"), diff --git a/tests/runtime/test_session_api.py b/tests/runtime/test_session_api.py index be91f5276..db34086f5 100644 --- a/tests/runtime/test_session_api.py +++ b/tests/runtime/test_session_api.py @@ -52,12 +52,12 @@ async def prompt_stream(self, text): @pytest.fixture def registry(monkeypatch): monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _ScriptedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _ScriptedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") # Lives in condor.runtime.binding now; patch it at the source so both the # bound and unbound resolution paths see the stub. monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) return session_module diff --git a/tests/runtime/test_session_creation_lock.py b/tests/runtime/test_session_creation_lock.py new file mode 100644 index 000000000..45e533b8b --- /dev/null +++ b/tests/runtime/test_session_creation_lock.py @@ -0,0 +1,196 @@ +"""Concurrent session creation is serialized per key (CORR-187). + +``get_or_create_session`` is a check-then-create that spans many awaits +(subprocess spawn, eager context prompt). Before the per-key creation lock, +two concurrent calls for the same key both spawned a full client and the +second registration overwrote the first, whose process tree leaked until bot +shutdown; the session budget had the same TOCTOU. These tests *race* real +concurrent calls — they do not merely assert that a lock object exists. +""" + +import asyncio + +import pytest + +from condor.runtime import SessionKey, SessionSpec +from condor.runtime import sessions as session_module + + +class _SlowClient: + """Client stub whose start() yields to the loop, opening the race window.""" + + instances: list = [] + started = 0 + stopped = 0 + # When set, start() blocks until the event fires (for the parallelism test). + start_gate: asyncio.Event | None = None + + def __init__(self, **kwargs): + self.alive = True + type(self).instances.append(self) + + async def start(self): + type(self).started += 1 + if type(self).start_gate is not None: + await type(self).start_gate.wait() + else: + # Yield a few times so a concurrent caller can reach the registry + # check while this creation is mid-flight. + for _ in range(3): + await asyncio.sleep(0) + + async def stop(self): + self.alive = False + type(self).stopped += 1 + + async def prompt(self, text): + return "" + + async def prompt_stream(self, text): + return + yield # pragma: no cover - makes this an async generator + + +@pytest.fixture +def registry(monkeypatch): + """Fresh module state with the subprocess client stubbed out.""" + _SlowClient.instances = [] + _SlowClient.started = 0 + _SlowClient.stopped = 0 + _SlowClient.start_gate = None + monkeypatch.setattr(session_module, "_sessions", {}) + monkeypatch.setattr(session_module, "_pending_creates", {}) + monkeypatch.setattr(session_module, "_creation_locks", {}) + monkeypatch.setattr(session_module, "_creation_lock_refs", {}) + monkeypatch.setattr("condor.acp.client.ACPClient", _SlowClient) + monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") + monkeypatch.setattr( + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] + ) + return session_module + + +def _spec(key: SessionKey, user_id: int = 42) -> SessionSpec: + return SessionSpec(key=str(key), agent_key="claude-code", user_id=user_id) + + +def _assert_no_orphans(registry): + """Every started client is either registered in _sessions or stopped.""" + registered = {s.client for s in registry._sessions.values()} + for client in _SlowClient.instances: + assert ( + client in registered or client.stopped or not client.alive + ), "orphaned client: started but neither registered nor stopped" + + +def test_concurrent_same_key_spawns_once(registry): + """Two racing creates for one key: one start(), one shared session.""" + key = SessionKey.web(42, "slot1") + + async def scenario(): + return await asyncio.gather( + session_module.get_or_create_session(_spec(key)), + session_module.get_or_create_session(_spec(key)), + ) + + first, second = asyncio.run(scenario()) + assert first is second + assert _SlowClient.started == 1 + assert len(registry._sessions) == 1 + assert registry._sessions[str(key)] is first + + +def test_racing_creates_leave_no_orphan(registry): + """N racing creates for one key never strand a started client.""" + key = SessionKey.web(42, "slot1") + + async def scenario(): + return await asyncio.gather( + *(session_module.get_or_create_session(_spec(key)) for _ in range(5)) + ) + + results = asyncio.run(scenario()) + assert len({id(s) for s in results}) == 1 + _assert_no_orphans(registry) + assert _SlowClient.started == 1 + + +def test_budget_holds_under_concurrent_distinct_creates(registry): + """Racing creates of distinct keys never exceed MAX_SESSIONS_PER_USER.""" + cap = session_module.MAX_SESSIONS_PER_USER + total = cap + 3 + + async def scenario(): + return await asyncio.gather( + *( + session_module.get_or_create_session( + _spec(SessionKey.web(42, f"slot{i}")) + ) + for i in range(total) + ), + return_exceptions=True, + ) + + results = asyncio.run(scenario()) + sessions = [r for r in results if isinstance(r, session_module.AgentSession)] + refused = [r for r in results if isinstance(r, session_module.SessionLimitReached)] + unexpected = [ + r + for r in results + if not isinstance( + r, (session_module.AgentSession, session_module.SessionLimitReached) + ) + ] + assert not unexpected + assert len(sessions) == cap + assert len(refused) == total - cap + assert len([s for s in registry._sessions.values() if s.user_id == 42]) == cap + # The refused creates never spawned anything, so nothing to orphan. + assert _SlowClient.started == cap + _assert_no_orphans(registry) + + +def test_distinct_keys_create_concurrently(registry): + """Per-key locks: creations under different keys are not serialized.""" + _SlowClient.start_gate = asyncio.Event() # replaced inside the loop + + async def scenario(): + gate = asyncio.Event() + _SlowClient.start_gate = gate + tasks = [ + asyncio.create_task( + session_module.get_or_create_session( + _spec(SessionKey.web(42, f"slot{i}")) + ) + ) + for i in range(2) + ] + # Both creations must reach start() while neither has finished — + # a global creation lock would leave the second waiting forever. + async with asyncio.timeout(2): + while _SlowClient.started < 2: + await asyncio.sleep(0) + gate.set() + return await asyncio.gather(*tasks) + + first, second = asyncio.run(scenario()) + assert first is not second + assert len(registry._sessions) == 2 + + +def test_fast_path_returns_without_spawning(registry): + """An existing matching session is returned with no new spawn.""" + key = SessionKey.web(42, "slot1") + + async def scenario(): + created = await session_module.get_or_create_session(_spec(key)) + again = await session_module.get_or_create_session(_spec(key)) + return created, again + + created, again = asyncio.run(scenario()) + assert created is again + assert _SlowClient.started == 1 + # The reservation bookkeeping is fully cleaned up after every create. + assert registry._pending_creates == {} + assert registry._creation_locks == {} + assert registry._creation_lock_refs == {} diff --git a/tests/runtime/test_session_facade.py b/tests/runtime/test_session_facade.py index 55ac3c69a..692b73a94 100644 --- a/tests/runtime/test_session_facade.py +++ b/tests/runtime/test_session_facade.py @@ -47,12 +47,12 @@ async def prompt(self, text): def registry(monkeypatch): """Empty registry with the subprocess + context machinery stubbed out.""" monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _FakeClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _FakeClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") # Lives in condor.runtime.binding now; patch it at the source so both the # bound and unbound resolution paths see the stub. monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) return session_module diff --git a/tests/runtime/test_wake.py b/tests/runtime/test_wake.py index 8ace40707..68a249330 100644 --- a/tests/runtime/test_wake.py +++ b/tests/runtime/test_wake.py @@ -101,10 +101,10 @@ def registry(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _EchoClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _EchoClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) # Sink factories and the in-flight counts are process-global, exactly like the # confirmation registry: reset both so tests cannot leak into each other. @@ -331,7 +331,7 @@ async def scenario(): @pytest.fixture def gated(registry, monkeypatch): - monkeypatch.setattr(session_module, "ACPClient", _GatedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _GatedClient) return registry diff --git a/tests/test_agents.py b/tests/test_agents.py index d77231274..4d61c8485 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -544,11 +544,11 @@ def _run_create_session(monkeypatch, *, chat_id, user_id): from condor.runtime import sessions as session_module monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _FakeACPClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _FakeACPClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") # Resolved through condor.runtime.binding now, so patch it at the source. monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) _FakeACPClient.last_extra_env = None spec = SessionSpec( diff --git a/tests/test_agents_chat_id_ownership.py b/tests/test_agents_chat_id_ownership.py new file mode 100644 index 000000000..d57231984 --- /dev/null +++ b/tests/test_agents_chat_id_ownership.py @@ -0,0 +1,265 @@ +"""Ownership gate on body-supplied ``chat_id`` push targets (SEC-198). + +``/agents/notify``, ``/agents/{slug}/delegate``, ``/agents/{slug}/consult`` and +the strategy start routes all forward a ``chat_id`` from the request body to +outbound Telegram sends (directly, or via the delegation's completion notice, +the consult's confirmation prompts, or the engine's tick notifications). The +routes already refuse ``req.user_id`` impersonation; this pins the same rule for +the outbound address: a caller reaches their own private chat for free, a group +only when Telegram confirms they are a member, and anything unverifiable is +refused with 403. Admins are exempt, mirroring SEC-081/SEC-152. + +Sync tests driving coroutines with ``asyncio.run`` (pytest-asyncio is not +installed in this venv), fakes in the style of test_notify_route.py. +""" + +import asyncio +from types import SimpleNamespace + +import pytest +from fastapi import HTTPException + +import config_manager +from condor.agents import delegate as delegate_module +from condor.web.models import WebUser +from condor.web.routes import agents as agents_routes +from condor.web.routes.agents import ( + ConsultRequest, + DelegateRequest, + NotifyRequest, + StartStrategyRequest, + _check_chat_access, + consult_agent, + delegate_agent, + notify_user, +) + +CALLER = WebUser(id=1, role="user") +ADMIN = WebUser(id=99, role="admin") +FOREIGN_CHAT = 555 # another user's private chat +GROUP_CHAT = -100123 # a Telegram group id + + +class _FakeConfigManager: + def is_admin(self, user_id: int) -> bool: + return user_id == ADMIN.id + + +class _FakeBot: + """Records sends; answers ``get_chat_member`` with a configurable shape.""" + + def __init__(self, member=None): + self.member = member + self.calls: list[dict] = [] + self.membership_checks: list[tuple] = [] + + async def send_message(self, **kw): + self.calls.append(kw) + return {"ok": True} + + async def get_chat_member(self, chat_id, user_id): + self.membership_checks.append((chat_id, user_id)) + if isinstance(self.member, Exception): + raise self.member + return self.member + + +@pytest.fixture(autouse=True) +def cm(monkeypatch): + monkeypatch.setattr(config_manager, "get_config_manager", _FakeConfigManager) + + +@pytest.fixture +def bot(monkeypatch): + fake = _FakeBot() + monkeypatch.setattr(delegate_module, "resolve_bot", lambda b=None: fake) + return fake + + +def _resolves_to(monkeypatch, conversation_id: str = ""): + async def fake(session_key: str) -> str: + return conversation_id if session_key else "" + + monkeypatch.setattr(agents_routes, "_conversation_for_session", fake) + + +# ── The check itself ── + + +def test_own_private_chat_needs_no_verification(bot): + asyncio.run(_check_chat_access(CALLER.id, CALLER.id)) + assert bot.membership_checks == [] + + +def test_zero_chat_id_passes(bot): + asyncio.run(_check_chat_access(CALLER.id, 0)) + assert bot.membership_checks == [] + + +def test_a_foreign_chat_is_refused_when_unverifiable(bot): + """The fake answers ``None`` — exactly what ``_HttpBot`` without a token + says. An unverifiable target must fail closed.""" + with pytest.raises(HTTPException) as exc: + asyncio.run(_check_chat_access(CALLER.id, FOREIGN_CHAT)) + assert exc.value.status_code == 403 + + +def test_a_telegram_error_fails_closed(bot): + bot.member = RuntimeError("Bad Request: user not found") + with pytest.raises(HTTPException) as exc: + asyncio.run(_check_chat_access(CALLER.id, GROUP_CHAT)) + assert exc.value.status_code == 403 + + +def test_a_verified_group_member_passes_object_shape(bot): + """A live python-telegram-bot answers a ChatMember with ``.status``.""" + bot.member = SimpleNamespace(status="member") + asyncio.run(_check_chat_access(CALLER.id, GROUP_CHAT)) + assert bot.membership_checks == [(GROUP_CHAT, CALLER.id)] + + +def test_a_verified_group_member_passes_envelope_shape(bot): + """``_HttpBot`` answers Telegram's raw envelope.""" + bot.member = {"ok": True, "result": {"status": "administrator"}} + asyncio.run(_check_chat_access(CALLER.id, GROUP_CHAT)) + + +def test_a_left_or_kicked_member_is_refused(bot): + for status in ("left", "kicked"): + bot.member = {"ok": True, "result": {"status": status}} + with pytest.raises(HTTPException) as exc: + asyncio.run(_check_chat_access(CALLER.id, GROUP_CHAT)) + assert exc.value.status_code == 403 + + +def test_admin_may_target_any_chat(bot): + """Admins route announcements into shared groups/channels (SEC-152).""" + asyncio.run(_check_chat_access(ADMIN.id, FOREIGN_CHAT)) + assert bot.membership_checks == [] + + +# ── POST /agents/notify ── + + +def test_notify_refuses_a_foreign_chat_before_any_side_effect(monkeypatch, bot): + _resolves_to(monkeypatch, "conv-1") + noted = [] + from condor.runtime import conversations + + monkeypatch.setattr( + conversations, "record_system", lambda *a, **kw: noted.append(a) + ) + + with pytest.raises(HTTPException) as exc: + asyncio.run( + notify_user( + NotifyRequest(text="phish", chat_id=FOREIGN_CHAT, session_key="w:1:s"), + user=CALLER, + ) + ) + + assert exc.value.status_code == 403 + assert bot.calls == [] # nothing left the process + assert noted == [] # the 403 lands before the transcript note + + +def test_notify_still_delivers_to_the_callers_own_chat(monkeypatch, bot): + _resolves_to(monkeypatch, "") + result = asyncio.run( + notify_user(NotifyRequest(text="done", chat_id=CALLER.id), user=CALLER) + ) + assert result["sent"] is True + assert bot.calls[0]["chat_id"] == CALLER.id + + +def test_notify_delivers_to_a_group_the_caller_belongs_to(monkeypatch, bot): + """The MCP crossback for a group session sends the group's id under the + member's own JWT — that legitimate path must keep working.""" + _resolves_to(monkeypatch, "") + bot.member = {"ok": True, "result": {"status": "member"}} + + result = asyncio.run( + notify_user(NotifyRequest(text="done", chat_id=GROUP_CHAT), user=CALLER) + ) + + assert result["sent"] is True + assert bot.calls[0]["chat_id"] == GROUP_CHAT + + +# ── POST /agents/{slug}/delegate ── + + +def _delegate(monkeypatch, req: DelegateRequest, user: WebUser): + monkeypatch.setattr(agents_routes, "_get_agent", lambda slug: SimpleNamespace()) + _resolves_to(monkeypatch, "") + started = [] + + async def fake_start(**kw): + started.append(kw) + return SimpleNamespace(task_id="t-1", status="running") + + monkeypatch.setattr(delegate_module, "start_delegation", fake_start) + return started, asyncio.run(delegate_agent("scout", req, user=user)) + + +def test_delegate_refuses_a_foreign_completion_chat(monkeypatch, bot): + with pytest.raises(HTTPException) as exc: + _delegate(monkeypatch, DelegateRequest(task="t", chat_id=FOREIGN_CHAT), CALLER) + assert exc.value.status_code == 403 + + +def test_delegate_accepts_the_callers_own_chat(monkeypatch, bot): + started, result = _delegate( + monkeypatch, DelegateRequest(task="t", chat_id=CALLER.id), CALLER + ) + assert result["task_id"] == "t-1" + assert started[0]["chat_id"] == CALLER.id + + +# ── POST /agents/{slug}/consult ── + + +def test_consult_refuses_a_foreign_chat(monkeypatch, bot): + from condor.agents import consult as consult_module + + called = [] + + async def fake_run_consult(**kw): # pragma: no cover - must not be reached + called.append(kw) + return "no" + + monkeypatch.setattr(consult_module, "run_consult", fake_run_consult) + + with pytest.raises(HTTPException) as exc: + asyncio.run( + consult_agent( + "scout", ConsultRequest(task="t", chat_id=FOREIGN_CHAT), user=CALLER + ) + ) + + assert exc.value.status_code == 403 + assert called == [] + + +# ── POST /agents/{slug}(/strategies/{sslug})/start ── + + +def test_start_refuses_a_foreign_notification_chat(monkeypatch, bot): + from condor.agents import config as config_module + + monkeypatch.setattr(config_module, "load_full_config", lambda *a, **kw: {}) + agent = SimpleNamespace(slug="scout") + strategy = SimpleNamespace( + slug="scalp", dir=".", default_config={}, default_trading_context="" + ) + + with pytest.raises(HTTPException) as exc: + asyncio.run( + agents_routes._start( + agent, + strategy, + StartStrategyRequest(chat_id=FOREIGN_CHAT), + CALLER.id, + ) + ) + assert exc.value.status_code == 403 diff --git a/tests/test_archived_cache.py b/tests/test_archived_cache.py new file mode 100644 index 000000000..961be03dd --- /dev/null +++ b/tests/test_archived_cache.py @@ -0,0 +1,167 @@ +"""Archived-bot performance cache is bounded and single-flight (PERF-185). + +condor/web/routes/archived.py used to keep two unbounded module-level dicts — +one full ``ArchivedBotPerformance`` (up to ~5000 PnL points) plus a duplicate +executor list per (server, db_path) ever viewed — for the process lifetime, and +its check-then-fetch had no in-flight coalescing, so concurrent cold-cache +requests each walked the entire paginated trade history independently. + +These pin the fix: a small LRU with eviction, one shared fetch per key for +concurrent callers, executors stored once (inside the performance entry), and +cached responses identical to fresh ones. +""" + +import asyncio + +from condor.web.routes import archived + + +class FakeArchivedBots: + """Counts backend calls; optional delay widens the concurrency window.""" + + def __init__(self, delay: float = 0.0): + self.delay = delay + self.summary_calls = 0 + self.trades_calls = 0 + self.executors_calls = 0 + + async def get_database_summary(self, db_path): + self.summary_calls += 1 + if self.delay: + await asyncio.sleep(self.delay) + return { + "bot_name": f"bot-{db_path}", + "total_trades": 0, + "total_orders": 0, + "trading_pairs": ["SOL-USDC"], + "exchanges": ["binance"], + } + + async def get_database_trades(self, db_path, limit=500, offset=0): + self.trades_calls += 1 + return {"trades": []} + + async def get_database_executors(self, db_path): + self.executors_calls += 1 + return { + "executors": [ + { + "id": "e1", + "connector": "binance", + "trading_pair": "SOL-USDC", + "side": "BUY", + "net_pnl_quote": 1.5, + "timestamp": 1700000000, + } + ] + } + + +class FakeClient: + def __init__(self, delay: float = 0.0): + self.archived_bots = FakeArchivedBots(delay=delay) + + +def _reset_caches(): + archived._performance_cache.clear() + archived._performance_inflight.clear() + + +def test_cache_is_bounded_lru(): + """Viewing cap+5 bots keeps at most cap entries; oldest are evicted.""" + _reset_caches() + cap = archived._PERFORMANCE_CACHE_MAX + client = FakeClient() + + async def run(): + for i in range(cap + 5): + await archived._fetch_and_cache_performance(client, "srv", f"db{i}.sqlite") + + asyncio.run(run()) + + assert len(archived._performance_cache) <= cap + # The five oldest were evicted, the newest survive + assert ("srv", "db0.sqlite") not in archived._performance_cache + assert ("srv", "db4.sqlite") not in archived._performance_cache + assert ("srv", f"db{cap + 4}.sqlite") in archived._performance_cache + _reset_caches() + + +def test_lru_hit_refreshes_recency(): + """A cache hit moves the entry to most-recently-used, deferring eviction.""" + _reset_caches() + cap = archived._PERFORMANCE_CACHE_MAX + client = FakeClient() + + async def run(): + for i in range(cap): + await archived._fetch_and_cache_performance(client, "srv", f"db{i}.sqlite") + # Touch the oldest entry, then overflow by one + await archived._fetch_and_cache_performance(client, "srv", "db0.sqlite") + await archived._fetch_and_cache_performance(client, "srv", "extra.sqlite") + + asyncio.run(run()) + + assert ("srv", "db0.sqlite") in archived._performance_cache # refreshed, kept + assert ( + "srv", + "db1.sqlite", + ) not in archived._performance_cache # now oldest, evicted + _reset_caches() + + +def test_concurrent_first_requests_share_one_fetch(): + """Two concurrent cold-cache requests issue exactly one backend walk.""" + _reset_caches() + client = FakeClient(delay=0.02) + + async def run(): + return await asyncio.gather( + archived._fetch_and_cache_performance(client, "srv", "db.sqlite"), + archived._fetch_and_cache_performance(client, "srv", "db.sqlite"), + ) + + first, second = asyncio.run(run()) + + assert client.archived_bots.summary_calls == 1 + assert client.archived_bots.trades_calls == 1 + assert client.archived_bots.executors_calls == 1 + assert first is second # both awaited the same task's result + assert not archived._performance_inflight # cleared when done + _reset_caches() + + +def test_executors_stored_once_inside_performance_entry(): + """No second executors cache: the perf entry is the single source.""" + _reset_caches() + client = FakeClient() + + perf = asyncio.run( + archived._fetch_and_cache_performance(client, "srv", "db.sqlite") + ) + + assert not hasattr(archived, "_executors_cache") + cached = archived._cache_get(("srv", "db.sqlite")) + assert cached is perf + assert cached.executors is perf.executors + assert len(perf.executors) == 1 + _reset_caches() + + +def test_cached_response_identical_to_fresh(): + """A cached answer matches a fresh fetch of the same db_path exactly.""" + _reset_caches() + fresh = asyncio.run( + archived._fetch_and_cache_performance(FakeClient(), "srv", "db.sqlite") + ) + cached = asyncio.run( + archived._fetch_and_cache_performance(FakeClient(), "srv", "db.sqlite") + ) + assert cached is fresh # served from cache, no refetch + + _reset_caches() + refetched = asyncio.run( + archived._fetch_and_cache_performance(FakeClient(), "srv", "db.sqlite") + ) + assert refetched.model_dump() == fresh.model_dump() + _reset_caches() diff --git a/tests/test_backtest_server_scope.py b/tests/test_backtest_server_scope.py new file mode 100644 index 000000000..335d50cfd --- /dev/null +++ b/tests/test_backtest_server_scope.py @@ -0,0 +1,80 @@ +"""SEC-197: backtest fetch/delete must be scoped to the {name} path server. + +The single-item endpoints used to resolve saved results purely by task_id, +so a user with access to server A could read or delete a backtest that +belonged to server B by requesting /servers/A/backtesting/...{task_id_from_B}. +These pin the fix: cross-server ids 404 and leave the record untouched, +while same-server fetch and delete keep working. +""" + +from __future__ import annotations + +import asyncio + +import pytest +from fastapi import HTTPException + +import condor.web.routes.backtesting as bt_routes +from condor.backtest_store import BacktestStore + + +class _NoLiveConfigManager: + """Config manager whose live API is unreachable, forcing the saved fallback.""" + + async def get_client(self, name): + raise RuntimeError("no live api in tests") + + +@pytest.fixture +def store(tmp_path, monkeypatch): + store = BacktestStore(data_dir=tmp_path / "backtests") + store.save_result("server-b", "task-b", {"status": "completed", "config": {}}) + monkeypatch.setattr(bt_routes, "get_backtest_store", lambda: store) + monkeypatch.setattr(bt_routes, "get_config_manager", lambda: _NoLiveConfigManager()) + return store + + +def test_get_cross_server_task_is_404(store): + with pytest.raises(HTTPException) as exc: + asyncio.run(bt_routes.get_backtest_task("server-a", "task-b", user=None)) + assert exc.value.status_code == 404 + + +def test_get_same_server_task_returns_saved_result(store): + result = asyncio.run(bt_routes.get_backtest_task("server-b", "task-b", user=None)) + assert result["saved"] is True + assert result["server"] == "server-b" + + +def test_delete_task_cross_server_is_404_and_keeps_record(store): + with pytest.raises(HTTPException) as exc: + asyncio.run(bt_routes.delete_backtest_task("server-a", "task-b", user=None)) + assert exc.value.status_code == 404 + assert store.get_result("task-b") is not None + + +def test_delete_task_same_server_deletes(store): + result = asyncio.run( + bt_routes.delete_backtest_task("server-b", "task-b", user=None) + ) + assert result == {"deleted": True} + assert store.get_result("task-b") is None + + +def test_delete_saved_cross_server_is_404_and_keeps_record(store): + with pytest.raises(HTTPException) as exc: + asyncio.run(bt_routes.delete_saved_result("server-a", "task-b", user=None)) + assert exc.value.status_code == 404 + assert store.get_result("task-b") is not None + + +def test_delete_saved_same_server_deletes(store): + result = asyncio.run(bt_routes.delete_saved_result("server-b", "task-b", user=None)) + assert result == {"deleted": True} + assert store.get_result("task-b") is None + + +def test_delete_saved_unknown_id_is_404(store): + with pytest.raises(HTTPException) as exc: + asyncio.run(bt_routes.delete_saved_result("server-b", "missing", user=None)) + assert exc.value.status_code == 404 diff --git a/tests/test_bot_mode_session_resolution.py b/tests/test_bot_mode_session_resolution.py index 513495f7b..98d879c63 100644 --- a/tests/test_bot_mode_session_resolution.py +++ b/tests/test_bot_mode_session_resolution.py @@ -1,4 +1,4 @@ -"""Bot-mode session→bot_name resolution (`_session_bot_base`). +"""Bot-mode session→bot_name resolution (`session_bot_base`). The per-session PnL distribution and the operator's live-executor view share this one helper, so its fallback rules must hold for both: a non-empty per-session @@ -11,7 +11,7 @@ import yaml -from condor.web.routes.agents import _session_bot_base +from condor.agents.attribution import session_bot_base def _write_session(strategy_dir: Path, num: int, cfg: dict) -> None: @@ -24,7 +24,7 @@ def test_per_session_bot_name_wins(tmp_path): # A session that recorded its runtime-derived deploy name takes precedence. _write_session(tmp_path, 1, {"bot_name": "dn-CL-BRENTOIL-mm-20260724-182221"}) assert ( - _session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 1) + session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 1) == "dn-CL-BRENTOIL-mm-20260724-182221" ) @@ -32,23 +32,23 @@ def test_per_session_bot_name_wins(tmp_path): def test_empty_session_bot_name_falls_back_to_default(tmp_path): # Early session saved bot_name as '' — must still resolve to the shared bot. _write_session(tmp_path, 2, {"bot_name": ""}) - assert _session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 2) == "dn-mm" + assert session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 2) == "dn-mm" def test_missing_session_bot_name_falls_back_to_default(tmp_path): # Session config predates the key entirely. _write_session(tmp_path, 3, {"some_other": 1}) - assert _session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 3) == "dn-mm" + assert session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 3) == "dn-mm" def test_no_session_dir_uses_default(tmp_path): # A session_num with no on-disk config still maps to the strategy default. - assert _session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 99) == "dn-mm" + assert session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 99) == "dn-mm" def test_direct_executor_strategy_resolves_empty(tmp_path): # No default and no per-session bot_name → direct-executor agent, nothing to # attribute; distribution and operator merge both correctly skip it. _write_session(tmp_path, 1, {"bot_name": ""}) - assert _session_bot_base(tmp_path, {}, 1) == "" - assert _session_bot_base(tmp_path, None, 99) == "" + assert session_bot_base(tmp_path, {}, 1) == "" + assert session_bot_base(tmp_path, None, 99) == "" diff --git a/tests/test_bot_ownership.py b/tests/test_bot_ownership.py index 861032a68..0eb35f04a 100644 --- a/tests/test_bot_ownership.py +++ b/tests/test_bot_ownership.py @@ -23,7 +23,7 @@ RiskState, auto_approve_with_risk_check, ) -from condor.agents.strategy import _slugify +from condor.frontmatter import slugify _OPTIONS = [{"kind": "allow_once", "optionId": "allow"}] NS = "brigado-ema_trend" @@ -50,9 +50,9 @@ def _drive(callback, call) -> str: def test_slugs_never_contain_the_delimiter(): - """The whole scheme rests on '-' being unambiguous: _slugify maps it to '_'.""" + """The whole scheme rests on '-' being unambiguous: slugify maps it to '_'.""" for name in ("RIVER Scalper v2", "ema-trend", " spot - perp ", "BTC/USD mm"): - assert "-" not in _slugify(name) + assert "-" not in slugify(name) def test_namespace_matches_base_tag_and_deployed_instance(): diff --git a/tests/test_bot_performance.py b/tests/test_bot_performance.py index 0684a8580..b0bd3d53f 100644 --- a/tests/test_bot_performance.py +++ b/tests/test_bot_performance.py @@ -19,6 +19,7 @@ from condor.fetchers.bot_performance import ( _aggregate_by_bot, bot_executor_rows, + clear_history_cache, clear_snapshot_cache, extract_snapshots, fetch_all_bot_performance, @@ -27,10 +28,12 @@ @pytest.fixture(autouse=True) def _no_snapshot_cache_bleed(): - """Isolate the per-server snapshot cache between tests.""" + """Isolate the per-server snapshot and history caches between tests.""" clear_snapshot_cache() + clear_history_cache() yield clear_snapshot_cache() + clear_history_cache() # ── Sample payloads ── @@ -637,6 +640,64 @@ def test_slice_history_sums_multiple_instances(): assert slice_history([h1, h2], 0, 12) == (2.0, 20.0, 1.0, 0.2) +def test_slice_history_series_matches_per_stamp_slices(): + """The merge-pass series is byte-identical to per-stamp slice_history.""" + from condor.fetchers.bot_performance import slice_history, slice_history_series + + # Single instance. + h1 = [ + (10.0, 1.0, 100.0, 1.0, 0.1), + (20.0, 3.0, 300.0, 4.0, 0.3), + (30.0, 5.0, 500.0, 9.0, 0.5), + ] + # Second instance with disjoint sample epochs, including rows earlier + # than the window's `since` (they fold into the baseline, not the curve). + h2 = [ + (5.0, 0.5, 50.0, 1.0, 0.05), + (12.0, 2.5, 250.0, 2.0, 0.25), + (27.0, 4.5, 450.0, 3.0, 0.45), + (44.0, 6.5, 650.0, 4.0, 0.65), + ] + + for histories, since, until in ( + ([h1], 0.0, 100.0), # single instance + ([h1, h2], 0.0, 100.0), # disjoint epochs merged + ([h1, h2], 11.0, 100.0), # rows earlier than since → nonzero baseline + ([h1, h2], 0.0, 28.0), # until cutoff drops later stamps + ): + stamps = sorted({t for h in histories for t, *_ in h if since <= t <= until}) + expected = [(t, *slice_history(histories, since, t)) for t in stamps] + assert slice_history_series(histories, since, stamps) == expected + + # Empty stamps and empty histories degrade cleanly. + assert slice_history_series([h1], 0.0, []) == [] + assert slice_history_series([], 0.0, [10.0]) == [(10.0, 0.0, 0.0, 0.0, 0.0)] + + +def test_slice_history_series_is_linear_not_quadratic(): + """3 instances × 5000 rows: one merge pass stays well under 100ms.""" + import time + + from condor.fetchers.bot_performance import slice_history_series + + histories = [] + for k in range(3): + histories.append( + [ + (float(k + i * 3), float(i), float(i * 10), float(i), float(i) / 10) + for i in range(5000) + ] + ) + stamps = sorted({t for h in histories for t, *_ in h}) + + t0 = time.perf_counter() + out = slice_history_series(histories, 0.0, stamps) + elapsed = time.perf_counter() - t0 + + assert len(out) == len(stamps) + assert elapsed < 0.1, f"series build took {elapsed:.3f}s" + + def test_fetch_instance_history_sums_controllers_and_filters_trades(): from condor.fetchers.bot_performance import fetch_instance_history @@ -976,6 +1037,159 @@ def test_history_forward_carry_survives_a_controller_that_stops_reporting(): assert [round(r[2], 4) for r in hist] == [100.0, 100.0, 100.0] +# ── PERF-184: instance-history cursor walks are cached + coalesced ── +# +# One session-detail request walks the same instance's history twice +# back-to-back (performance rollup, then PnL series), and the tick engine +# repeats the walk every tick while the dashboard polls the same strategy. The +# cache collapses those to one walk per (server, instance, interval) per TTL. + + +class _CountingHistory: + def __init__(self, rows, delay=0.0, fail_first=False): + self.rows = rows + self.delay = delay + self.fail_first = fail_first + self.calls = 0 + + async def get_controller_performance_history(self, **_kw): + self.calls += 1 + if self.delay: + await asyncio.sleep(self.delay) + if self.fail_first and self.calls == 1: + raise RuntimeError("api down") + return {"data": self.rows} + + +def _history_server_client(base_url, rows, **kw): + return SimpleNamespace( + base_url=base_url, bot_orchestration=_CountingHistory(rows, **kw) + ) + + +_HIST_ROWS = [ + _hist_row("2026-08-06T22:00:00+00:00", "btc", 1.0, 10.0), + _hist_row("2026-08-06T22:15:00+00:00", "btc", 2.0, 20.0), +] + + +def test_repeated_history_reads_within_ttl_share_one_walk(): + """The two walks inside one session-detail request collapse to one.""" + from condor.fetchers.bot_performance import fetch_instance_history + + client = _history_server_client("http://hist-a:8000", _HIST_ROWS) + + async def _go(): + first = await fetch_instance_history(client, "bot-1") + second = await fetch_instance_history(client, "bot-1") + return first, second + + first, second = asyncio.run(_go()) + assert client.bot_orchestration.calls == 1 + assert first == second + assert [r[1] for r in first] == [1.0, 2.0] + + +def test_concurrent_history_callers_share_one_inflight_walk(): + """Engine tick + dashboard poll arriving together join one cursor walk.""" + from condor.fetchers.bot_performance import fetch_instance_history + + client = _history_server_client("http://hist-a:8000", _HIST_ROWS, delay=0.02) + + async def _go(): + return await asyncio.gather( + *[fetch_instance_history(client, "bot-1") for _ in range(5)] + ) + + results = asyncio.run(_go()) + assert client.bot_orchestration.calls == 1 + assert all(len(r) == 2 for r in results) + + +def test_distinct_instances_and_intervals_never_share_a_walk(): + from condor.fetchers.bot_performance import fetch_instance_history + + client = _history_server_client("http://hist-a:8000", _HIST_ROWS) + + async def _go(): + await fetch_instance_history(client, "bot-1", interval="5m") + await fetch_instance_history(client, "bot-2", interval="5m") + await fetch_instance_history(client, "bot-1", interval="15m") + + asyncio.run(_go()) + assert client.bot_orchestration.calls == 3 + + +def test_failed_history_walk_is_not_cached(): + """The [] degrade on API error stays retryable — waiters share the failure.""" + from condor.fetchers.bot_performance import fetch_instance_history + + client = _history_server_client( + "http://flaky-hist:8000", _HIST_ROWS, delay=0.02, fail_first=True + ) + + async def _go(): + # Both waiters join the failing walk and both degrade to []. + failed = await asyncio.gather( + *[fetch_instance_history(client, "bot-1") for _ in range(2)] + ) + # Nothing was cached, so the next call really re-walks and succeeds. + recovered = await fetch_instance_history(client, "bot-1") + return failed, recovered + + failed, recovered = asyncio.run(_go()) + assert failed == [[], []] + assert client.bot_orchestration.calls == 2 + assert [r[1] for r in recovered] == [1.0, 2.0] + + +def test_history_client_without_base_url_bypasses_the_cache(): + from condor.fetchers.bot_performance import fetch_instance_history + + client = _history_server_client("", _HIST_ROWS) + + async def _go(): + await fetch_instance_history(client, "bot-1") + await fetch_instance_history(client, "bot-1") + + asyncio.run(_go()) + assert client.bot_orchestration.calls == 2 + + +def test_history_cache_is_bounded_lru(monkeypatch): + """Stopped instances must not pin their pages forever.""" + import condor.fetchers.bot_performance as bp + + monkeypatch.setattr(bp, "_HISTORY_CACHE_MAX", 2) + client = _history_server_client("http://hist-a:8000", _HIST_ROWS) + + async def _go(): + await bp.fetch_instance_history(client, "bot-1") + await bp.fetch_instance_history(client, "bot-2") + await bp.fetch_instance_history(client, "bot-3") # evicts bot-1 + assert len(bp._history_cache) == 2 + await bp.fetch_instance_history(client, "bot-1") # miss: re-walks + + asyncio.run(_go()) + assert client.bot_orchestration.calls == 4 + + +def test_history_inflight_from_another_loop_is_not_awaited(): + """A task created on one event loop is never awaited from another.""" + import condor.fetchers.bot_performance as bp + + client = _history_server_client("http://hist-a:8000", _HIST_ROWS) + key = ("http://hist-a:8000", "bot-1", "5m", 500, 5000) + # A stale in-flight entry left by a different loop: the guard must ignore it + # (awaiting a foreign loop's task raises) and start its own walk. + foreign = SimpleNamespace() # never awaited — would blow up if it were + bp._history_inflight[key] = (object(), foreign) + + hist = asyncio.run(bp.fetch_instance_history(client, "bot-1")) + assert client.bot_orchestration.calls == 1 + assert [r[1] for r in hist] == [1.0, 2.0] + + def test_partition_includes_archived_instances(): """A stopped bot's realized PnL is exactly what its session is judged on.""" from condor.fetchers.bot_performance import partition_instances diff --git a/tests/test_chat_ws_conversations.py b/tests/test_chat_ws_conversations.py index a88066758..98304fb72 100644 --- a/tests/test_chat_ws_conversations.py +++ b/tests/test_chat_ws_conversations.py @@ -68,10 +68,10 @@ def runtime_env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _ScriptedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _ScriptedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) return tmp_path diff --git a/tests/test_chat_ws_prewarm.py b/tests/test_chat_ws_prewarm.py index 1c13167f9..75073b7b9 100644 --- a/tests/test_chat_ws_prewarm.py +++ b/tests/test_chat_ws_prewarm.py @@ -71,12 +71,12 @@ def runtime_env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _SlowClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _SlowClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr(chat_ws, "_pending_spawns", {}) monkeypatch.setattr(chat_ws, "_active_prompt_tasks", {}) monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) return tmp_path diff --git a/tests/test_chat_ws_steering.py b/tests/test_chat_ws_steering.py index ec7ba80d0..a8c447837 100644 --- a/tests/test_chat_ws_steering.py +++ b/tests/test_chat_ws_steering.py @@ -87,10 +87,10 @@ def ws_env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _GatedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _GatedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) monkeypatch.setattr(chat_ws, "_active_prompt_tasks", {}) diff --git a/tests/test_chat_ws_wake.py b/tests/test_chat_ws_wake.py index c6470f7da..8bfc2b7f0 100644 --- a/tests/test_chat_ws_wake.py +++ b/tests/test_chat_ws_wake.py @@ -61,10 +61,10 @@ def ws_env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _EchoClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _EchoClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) monkeypatch.setattr(chat_ws, "_active_prompt_tasks", {}) diff --git a/tests/test_custom_provider.py b/tests/test_custom_provider.py index 280f86f70..9ba452bf6 100644 --- a/tests/test_custom_provider.py +++ b/tests/test_custom_provider.py @@ -188,6 +188,24 @@ def test_resolve_custom_endpoint_passes_through_other_keys(): assert resolve_custom_endpoint("custom@Missing:m", user_data={}) == (None, None) +def test_resolve_custom_endpoint_strict_raises_for_missing_named_endpoint(): + # The chat-session path (sessions.py) wants a loud, actionable failure + # instead of the lenient warn-and-env-fallback default. + with pytest.raises(RuntimeError, match="No saved endpoint named"): + resolve_custom_endpoint("custom@Missing:m", user_data={}, strict=True) + + +def test_resolve_custom_endpoint_strict_resolves_saved_endpoint(): + # strict only changes the missing-endpoint case; a saved one still resolves + user_data = {} + save_custom_provider(user_data, "Venice", "https://x.example/v1", "sk-1") + key = build_custom_agent_key("Venice", "claude-sonnet-4-6") + assert resolve_custom_endpoint(key, user_data=user_data, strict=True) == ( + "https://x.example/v1", + "sk-1", + ) + + def test_resolve_custom_endpoint_env_fallback(monkeypatch): monkeypatch.setenv("CUSTOM_LLM_BASE_URL", "https://env.example/v1") monkeypatch.setenv("CUSTOM_LLM_API_KEY", "sk-env") diff --git a/tests/test_default_server_sync.py b/tests/test_default_server_sync.py new file mode 100644 index 000000000..55785d9ad --- /dev/null +++ b/tests/test_default_server_sync.py @@ -0,0 +1,216 @@ +"""A default server picked on one surface is the default on the other. + +Both surfaces write ``chat_defaults`` in config.yml when the user picks a +default. Telegram *also* writes the choice into its pickle ``user_data``, and +``get_effective_server`` used to read that pickle first — so a default set in the +web dashboard was invisible to a live Telegram session: config.yml said one +server, the in-memory pickle said the one chosen before it, and the pickle won +every lookup until the bot process restarted. Logging out and back in on the web +made the web agree with itself; nothing made Telegram agree with the web. + +So config.yml is read first now, and the pickle is kept as a cache of it. The one +thing that must survive the inversion is a *pinned* server: a routine or agent run +launched against a named server must not be re-resolved to the chat's default. +""" + +import asyncio + +import pytest + +import config_manager as cm_module +from condor.preferences import SERVER_PIN_KEY, USER_PREFERENCES_KEY, set_active_server +from condor.routine_store import WebRoutineContext + +CHAT_ID = 4242 +WEB_PICK = "brigado_2" +TELEGRAM_PICK = "local" + + +class FakeConfigManager: + """Only the state ``get_effective_server`` reads, plus the preference writes.""" + + def __init__(self): + self._data = { + "servers": {WEB_PICK: {}, TELEGRAM_PICK: {}}, + "chat_defaults": {}, + } + self.preference_writes: list[tuple[int, str]] = [] + + def set_user_preference(self, user_id, key, value): + self.preference_writes.append((user_id, key)) + + def get_user_preferences(self, user_id): + return {} + + +@pytest.fixture +def cm(monkeypatch): + fake = FakeConfigManager() + monkeypatch.setattr(cm_module, "get_config_manager", lambda: fake) + return fake + + +def _telegram_user_data(server: str) -> dict: + """What a live Telegram session holds after the user picked ``server``.""" + user_data: dict = {} + set_active_server(user_data, server) + return user_data + + +def test_a_default_set_on_the_web_reaches_a_live_telegram_session(cm): + user_data = _telegram_user_data(TELEGRAM_PICK) + + # The web dashboard's "set default": config.yml only, no access to the pickle. + cm._data["chat_defaults"][CHAT_ID] = WEB_PICK + + assert cm_module.get_effective_server(CHAT_ID, user_data) == WEB_PICK + + +def test_the_pickle_is_corrected_rather_than_left_disagreeing(cm): + user_data = _telegram_user_data(TELEGRAM_PICK) + cm._data["chat_defaults"][CHAT_ID] = WEB_PICK + + cm_module.get_effective_server(CHAT_ID, user_data) + + assert user_data[USER_PREFERENCES_KEY]["general"]["active_server"] == WEB_PICK + + +def test_agreeing_with_config_costs_no_write(cm): + # This runs on every server lookup, and the setter persists the whole + # preference section — so a pickle that already agrees must stay silent. + user_data = _telegram_user_data(WEB_PICK) + user_data["_user_id"] = 7 + cm._data["chat_defaults"][CHAT_ID] = WEB_PICK + cm.preference_writes.clear() + + cm_module.get_effective_server(CHAT_ID, user_data) + + assert cm.preference_writes == [] + + +def test_a_chat_with_no_default_still_uses_the_users_own_pick(cm): + user_data = _telegram_user_data(TELEGRAM_PICK) + + assert cm_module.get_effective_server(CHAT_ID, user_data) == TELEGRAM_PICK + + +def test_a_deleted_server_named_in_config_does_not_win(cm): + user_data = _telegram_user_data(TELEGRAM_PICK) + cm._data["chat_defaults"][CHAT_ID] = "deleted" + + assert cm_module.get_effective_server(CHAT_ID, user_data) == TELEGRAM_PICK + + +def test_a_run_pinned_to_a_server_is_never_re_resolved(cm): + # A routine launched against WEB_PICK from a chat whose default is + # TELEGRAM_PICK: the launch server is the answer, or the run files its + # results under a server it never touched. + cm._data["chat_defaults"][CHAT_ID] = TELEGRAM_PICK + ctx = WebRoutineContext(WEB_PICK, bot=object(), chat_id=CHAT_ID) + + assert ctx.user_data[SERVER_PIN_KEY] is True + assert cm_module.get_effective_server(CHAT_ID, ctx.user_data) == WEB_PICK + + +def test_nothing_configured_resolves_to_nothing(cm): + assert cm_module.get_effective_server(CHAT_ID, {}) is None + assert cm_module.get_effective_server(CHAT_ID, None) is None + + +# --------------------------------------------------------------------------- +# The gate that keeps the pickle honest for handlers that never resolve. +# +# Most handlers (/portfolio, /bots, the gateway screens) read the pickle copy +# via ``get_active_server`` instead of resolving through +# ``get_effective_server``. Nothing corrected that copy on their path, so a +# default set on the web stayed invisible to /portfolio until the user happened +# to open /servers, which resolves as a side effect. ``@restricted`` runs on +# every Telegram update, so the correction belongs there. +# --------------------------------------------------------------------------- + + +class _FakeChat: + def __init__(self, chat_id): + self.id = chat_id + + +class _FakeUser: + id = 7 + username = "fede" + + +class _FakeUpdate: + def __init__(self, chat_id): + self.effective_chat = _FakeChat(chat_id) + self.effective_user = _FakeUser() + self.message = None + self.callback_query = None + + +class _FakeContext: + def __init__(self, user_data): + self.user_data = user_data + + +@pytest.fixture +def auth(monkeypatch, cm): + """``utils.auth`` with the config manager it imported swapped for the fake.""" + import utils.auth as auth_module + from config_manager import UserRole + + cm.get_user_role = lambda user_id: UserRole.USER + monkeypatch.setattr(auth_module, "get_config_manager", lambda: cm) + return auth_module + + +def test_a_handler_reading_the_pickle_sees_the_new_default_on_the_next_command( + auth, cm +): + """What /portfolio does: read ``get_active_server``, never resolve.""" + from condor.preferences import get_active_server + + user_data = _telegram_user_data(TELEGRAM_PICK) + seen = [] + + @auth.restricted + async def portfolio_like(update, context): + seen.append(get_active_server(context.user_data)) + + cm._data["chat_defaults"][CHAT_ID] = WEB_PICK # the web dashboard's pick + asyncio.run(portfolio_like(_FakeUpdate(CHAT_ID), _FakeContext(user_data))) + + assert seen == [WEB_PICK] + + +def test_a_pinned_run_is_not_rewritten_by_the_gate(auth, cm): + from condor.preferences import get_active_server + + cm._data["chat_defaults"][CHAT_ID] = TELEGRAM_PICK + ctx = WebRoutineContext(WEB_PICK, bot=object(), chat_id=CHAT_ID) + + @auth.restricted + async def handler(update, context): + return None + + asyncio.run(handler(_FakeUpdate(CHAT_ID), ctx)) + + assert get_active_server(ctx.user_data) == WEB_PICK + + +def test_the_refresh_never_blocks_the_command(auth, cm, monkeypatch): + import config_manager as broken + + monkeypatch.setattr( + broken, + "get_config_manager", + lambda: (_ for _ in ()).throw(RuntimeError("config unreadable")), + ) + ran = [] + + @auth.restricted + async def handler(update, context): + ran.append(True) + + asyncio.run(handler(_FakeUpdate(CHAT_ID), _FakeContext({}))) + + assert ran == [True] diff --git a/tests/test_delegate_on_complete.py b/tests/test_delegate_on_complete.py index 988ab0eda..068639cf2 100644 --- a/tests/test_delegate_on_complete.py +++ b/tests/test_delegate_on_complete.py @@ -372,7 +372,9 @@ async def scenario(): "scout", DelegateRequest( task="scan pools", - chat_id=42, + # The caller's own chat: a foreign chat_id is 403 since SEC-198 + # (see test_agents_chat_id_ownership.py). + chat_id=CALLER.id, session_key=session_key, on_complete=on_complete, ), diff --git a/tests/test_delegate_worker.py b/tests/test_delegate_worker.py index f510030e7..5ed5e333b 100644 --- a/tests/test_delegate_worker.py +++ b/tests/test_delegate_worker.py @@ -305,10 +305,10 @@ async def prompt(self, text): return "done" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", fake_build + "condor.runtime.toolsets.build_mcp_servers_for_session", fake_build ) monkeypatch.setattr( - "handlers.agents._shared.build_agent_context", lambda *a, **k: "prompt" + "condor.runtime.context.build_agent_context", lambda *a, **k: "prompt" ) monkeypatch.setattr(acp_client_module, "ACPClient", _FakeClient) diff --git a/tests/test_dex_liquidity_bins_route.py b/tests/test_dex_liquidity_bins_route.py index 31873f9ef..88e3510b5 100644 --- a/tests/test_dex_liquidity_bins_route.py +++ b/tests/test_dex_liquidity_bins_route.py @@ -1,9 +1,9 @@ """Liquidity bins for the depth column beside a pool's chart (FEAT-045). -Two things are pinned here. First, that ``fetch_liquidity_bins`` can be handed a -client — the web side holds a server *name*, not a chat id, and the -``get_pool_info`` → ``get_pools`` fallback for pools outside Gateway's DLMM list -must keep working through that door. Second, that the route degrades: an +Two things are pinned here. First, that ``fetch_liquidity_bins`` works through an +injected client — resolving one is the caller's job, and the ``get_pool_info`` → +``get_pools`` fallback for pools outside Gateway's DLMM list must keep working +through that door. Second, that the route degrades: an unsupported venue, an unreachable server and an upstream raise are all states the workspace renders, never a 502 the user cannot act on. """ @@ -117,14 +117,10 @@ def bins_url(connector="meteora", network="solana-mainnet-beta", pool=POOL): return f"/servers/srv/dex/pools/{pool}/bins?network={network}&connector={connector}" -# ── fetch_liquidity_bins takes a client ── +# ── fetch_liquidity_bins requires a client ── -def test_an_injected_client_replaces_the_telegram_accessor(monkeypatch): - def boom(*a, **kw): - raise AssertionError("get_client must not be reached when a client is given") - - monkeypatch.setattr(pool_data, "get_client", boom) +def test_an_injected_client_reaches_gateway(): clmm = FakeClmm(pool_info()) bins, info, error = run( pool_data.fetch_liquidity_bins(POOL, "meteora", client=FakeClient(clmm)) @@ -135,34 +131,25 @@ def boom(*a, **kw): assert clmm.calls[0][0] == "info" -def test_the_telegram_path_still_resolves_its_own_client(monkeypatch): - clmm = FakeClmm(pool_info()) - seen = {} +def test_a_missing_client_degrades_without_a_raise(): + bins, info, error = run(pool_data.fetch_liquidity_bins(POOL, "meteora")) + assert (bins, info) == (None, None) + assert "client" in error.lower() - async def fake_get_client(chat_id, context=None): - seen["chat_id"] = chat_id - return FakeClient(clmm) - monkeypatch.setattr(pool_data, "get_client", fake_get_client) - user_data = {} - bins, _info, error = run( - pool_data.fetch_liquidity_bins(POOL, "meteora", user_data=user_data, chat_id=7) +def test_two_viewers_share_one_gateway_call(): + # Distinct clients — a chat's and a dashboard viewer's — because the cache is + # keyed by pool, never by who is asking: the second read costs no Gateway call. + first_clmm, second_clmm = FakeClmm(pool_info()), FakeClmm(pool_info()) + first = run( + pool_data.fetch_liquidity_bins(POOL, "meteora", client=FakeClient(first_clmm)) + ) + second = run( + pool_data.fetch_liquidity_bins(POOL, "meteora", client=FakeClient(second_clmm)) ) - assert error is None and len(bins) == 3 - assert seen["chat_id"] == 7 - # Still cached in the caller's conversation, not in the process-wide dict. - assert pool_data._pool_bins_cache == {} - run(pool_data.fetch_liquidity_bins(POOL, "meteora", user_data=user_data, chat_id=7)) - assert len(clmm.calls) == 1 - - -def test_two_clientless_viewers_share_one_gateway_call(): - clmm = FakeClmm(pool_info()) - client = FakeClient(clmm) - first = run(pool_data.fetch_liquidity_bins(POOL, "meteora", client=client)) - second = run(pool_data.fetch_liquidity_bins(POOL, "meteora", client=client)) assert first[0] == second[0] - assert len(clmm.calls) == 1 + assert len(first_clmm.calls) == 1 + assert second_clmm.calls == [] def test_the_get_pools_fallback_survives_the_client_parameter(): diff --git a/tests/test_dex_upstream_status.py b/tests/test_dex_upstream_status.py new file mode 100644 index 000000000..16e4af172 --- /dev/null +++ b/tests/test_dex_upstream_status.py @@ -0,0 +1,155 @@ +"""The DEX panel is told when GeckoTerminal is throttling Condor. + +Everything the panel draws — pool rows, TVL, a pool page, its chart — comes from +one per-IP GeckoTerminal budget, and every failure behind it degrades to an empty +list. Without a state to read, "this chain has no pools" and "the budget is spent, +ask again in ten seconds" arrive at the browser looking identical. + +Two shapes are pinned here. The refusal *counters*, because a spent per-minute +budget refuses without opening the breaker — ``rate_limited`` stays False while +the rows are still missing, so only a counter delta proves it happened. And the +pool lookup answering 503 rather than 404 when it was refused: one of those is +worth retrying and the other never resolves. +""" + +import asyncio + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.dex as dex_routes +from condor import pool_data +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +POOL = "8sLbNZoA1cfnvMJLPfp98ZLAnFSYCFApfJKMbiXNLwxj" +USER = WebUser(id=222, username="u", first_name="U", role="user") + + +class FakeConfigManager: + def has_server_access(self, user_id, name, *a, **kw): + return True + + async def get_client(self, name): + raise AssertionError("no Gateway call belongs in these paths") + + +@pytest.fixture +def client(monkeypatch): + cm = FakeConfigManager() + monkeypatch.setattr(dex_routes, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + app = FastAPI() + app.include_router(dex_routes.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app) + + +@pytest.fixture(autouse=True) +def _clean_budget(): + pool_data.reset_gecko_throttle() + yield + pool_data.reset_gecko_throttle() + + +def test_a_healthy_budget_reports_itself_as_not_limited(client): + body = client.get("/servers/srv/dex/upstream").json() + + assert body["rate_limited"] is False + assert body["retry_in"] == 0 + assert body["budget"] == pool_data._GECKO_RATE_LIMIT + + +def test_an_open_breaker_reports_a_cooldown_to_count_down(client): + pool_data._trip_breaker() + + body = client.get("/servers/srv/dex/upstream").json() + + assert body["rate_limited"] is True + assert 0 < body["retry_in"] <= pool_data._GECKO_COOLDOWN + assert body["count_429"] == 1 + + +def test_the_counters_move_when_a_request_is_refused_without_a_429(client): + """A spent budget refuses silently — the counter is the only evidence.""" + before = client.get("/servers/srv/dex/upstream").json() + + # Spend the minute, then ask for one more slot than it holds. + async def spend(): + for _ in range(pool_data._GECKO_RATE_LIMIT): + await pool_data._acquire_rate_slot() + with pytest.raises(pool_data.GeckoRateLimited): + await pool_data._acquire_rate_slot() + + asyncio.run(spend()) + + after = client.get("/servers/srv/dex/upstream").json() + assert after["rate_limited"] is False, "a spent budget must not open the breaker" + assert after["throttled_calls"] > before["throttled_calls"] + assert after["requests_last_minute"] == pool_data._GECKO_RATE_LIMIT + + +def test_an_empty_page_says_whether_the_fetch_was_refused(client, monkeypatch): + async def refused(*a, **kw): + pool_data._trip_breaker() + return {"pools": [], "has_more": False} + + monkeypatch.setattr(pool_data, "list_gecko_pools_page", refused) + + body = client.get("/servers/srv/dex/pools?source=gecko").json() + + assert body["pools"] == [] + assert body["upstream"]["throttled_request"] is True + assert body["upstream"]["rate_limited"] is True + + +def test_a_page_that_simply_has_no_rows_is_not_blamed_on_the_budget( + client, monkeypatch +): + async def empty(*a, **kw): + return {"pools": [], "has_more": False} + + monkeypatch.setattr(pool_data, "list_gecko_pools_page", empty) + + body = client.get("/servers/srv/dex/pools?source=gecko").json() + + assert body["upstream"]["throttled_request"] is False + assert body["upstream"]["rate_limited"] is False + + +def test_a_refused_pool_lookup_is_a_503_the_browser_can_retry(client, monkeypatch): + async def refused(*a, **kw): + pool_data._trip_breaker() + return None + + monkeypatch.setattr(pool_data, "fetch_pool_by_address", refused) + + res = client.get(f"/servers/srv/dex/pools/{POOL}") + + assert res.status_code == 503 + assert "rate limit" in res.json()["detail"].lower() + + +def test_an_unknown_pool_is_still_a_404(client, monkeypatch): + async def missing(*a, **kw): + return None + + monkeypatch.setattr(pool_data, "fetch_pool_by_address", missing) + + res = client.get(f"/servers/srv/dex/pools/{POOL}") + + assert res.status_code == 404 + + +def test_favourites_carry_the_same_state_as_the_browser(client, monkeypatch): + async def refused(*a, **kw): + pool_data._trip_breaker() + return [] + + monkeypatch.setattr(pool_data, "fetch_pools_by_addresses", refused) + + body = client.get(f"/servers/srv/dex/pools-by-address?addresses={POOL}").json() + + assert body["pools"] == [] + assert body["upstream"]["throttled_request"] is True diff --git a/tests/test_gateway_network_config_redaction.py b/tests/test_gateway_network_config_redaction.py new file mode 100644 index 000000000..7d9938e59 --- /dev/null +++ b/tests/test_gateway_network_config_redaction.py @@ -0,0 +1,137 @@ +"""Gateway network config reads redact RPC credentials for non-owners (SEC-197). + +Reading gateway state stays at TRADER (SEC-166 pinned that line: a trader must +see what they trade against), but the network config's ``nodeURL`` embeds the +owner's paid RPC provider API key — Helius puts it in the query string, +Infura/Alchemy in the path. GET /settings/gateway/networks/{id} therefore +masks every secret-bearing part of URL-shaped values server-side for anyone +below the OWNER line, with the same admin bypass `_require_owner` grants. +The secret must never reach a non-owner's browser to be hidden in the UI. +""" + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.settings as settings_routes +from condor.web.auth import get_current_user +from condor.web.models import WebUser +from config_manager import ServerPermission + +SERVER = "alpha" +OWNER = WebUser(id=1, username="owner", first_name="O", role="user") +TRADER = WebUser(id=2, username="trader", first_name="T", role="user") +ADMIN = WebUser(id=3, username="admin", first_name="A", role="admin") + +NETWORK_ID = "solana-mainnet-beta" + +# One URL per place a provider hides the key, plus values that must survive. +CONFIG = { + "nodeURL": "https://mainnet.helius-rpc.com/?api-key=super-secret-key", + "wsURL": "wss://eth-mainnet.g.alchemy.com/v2/another-secret", + "basicAuthURL": "https://user:pass@rpc.example.com", + "bareURL": "https://api.mainnet-beta.solana.com", + "chain_id": 101, + "native_currency_symbol": "SOL", + "retries": {"docsURL": "https://mainnet.infura.io/v3/path-secret"}, +} + + +class FakeGateway: + async def get_network_config(self, network_id): + return dict(CONFIG) + + +class FakeClient: + def __init__(self): + self.gateway = FakeGateway() + + +class FakeConfigManager: + """Owner id 1, trader id 2 shared, admin id 3 with no grant on the server.""" + + def __init__(self, client): + self._client = client + + def get_server_permission(self, user_id, server_name): + if server_name != SERVER: + return None + if user_id == OWNER.id: + return ServerPermission.OWNER + if user_id == TRADER.id: + return ServerPermission.TRADER + return None + + def has_server_access(self, user_id, server_name, min_permission=None): + if self.is_admin(user_id): + return server_name == SERVER + return self.get_server_permission(user_id, server_name) is not None + + def is_admin(self, user_id): + return user_id == ADMIN.id + + async def get_client(self, server_name): + return self._client + + +@pytest.fixture +def env(monkeypatch): + client = FakeClient() + cm = FakeConfigManager(client) + monkeypatch.setattr(settings_routes, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + app = FastAPI() + app.include_router(settings_routes.router) + return app + + +def as_user(app, user): + app.dependency_overrides[get_current_user] = lambda: user + return TestClient(app) + + +def get_config(app, user): + resp = as_user(app, user).get( + f"/settings/gateway/networks/{NETWORK_ID}", params={"server": SERVER} + ) + assert resp.status_code == 200 + return resp.json() + + +def test_trader_never_receives_credential_bearing_url_parts(env): + """Query keys, path keys and userinfo are all masked before the wire.""" + body = get_config(env, TRADER) + config = body["config"] + + assert body["redacted"] is True + # Helius-style query-string key. + assert config["nodeURL"] == "https://mainnet.helius-rpc.com/…" + assert "super-secret-key" not in str(config) + # Alchemy/Infura-style path key — including one nested a level down. + assert config["wsURL"] == "wss://eth-mainnet.g.alchemy.com/…" + assert "another-secret" not in str(config) + assert config["retries"]["docsURL"] == "https://mainnet.infura.io/…" + assert "path-secret" not in str(config) + # Userinfo credentials. + assert config["basicAuthURL"] == "https://rpc.example.com/…" + assert "pass" not in config["basicAuthURL"] + + +def test_trader_still_sees_the_endpoint_host_and_plain_values(env): + """Redaction hides the secret, not the network: hosts and scalars survive.""" + config = get_config(env, TRADER)["config"] + + # A bare endpoint carries no secret and stays fully readable. + assert config["bareURL"] == "https://api.mainnet-beta.solana.com" + assert config["chain_id"] == 101 + assert config["native_currency_symbol"] == "SOL" + + +@pytest.mark.parametrize("user", [OWNER, ADMIN], ids=["owner", "admin"]) +def test_owner_line_reads_full_values(env, user): + """Owners edit these values, so they receive them verbatim; admins keep + the bypass `_require_owner` grants everywhere else in this module.""" + body = get_config(env, user) + + assert body["redacted"] is False + assert body["config"] == CONFIG diff --git a/tests/test_gateway_tokens.py b/tests/test_gateway_tokens.py new file mode 100644 index 000000000..be59a8dc0 --- /dev/null +++ b/tests/test_gateway_tokens.py @@ -0,0 +1,342 @@ +"""A pool's tokens reach Gateway's token list on their own. + +Gateway resolves a swap by mint address, but not a balance: it skips token +accounts whose mint is not on the network's token list, so an unregistered token +reads as 0 and every panel that sizes an order off a balance is working from an +empty wallet. Registration therefore has to happen without anyone asking for it. + +Three shapes are pinned here. That an already-listed token is never re-saved (the +memo is what makes opening a pool twice free). That a save is *verified* by +re-reading the list, because Gateway answers 200 and writes nothing when another +token already holds the ticker — the one failure that stays silent otherwise. And +that a failure is never remembered, so it stays retryable on the next visit. +""" + +import asyncio + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.dex as dex_routes +from condor.fetchers import gateway_tokens +from condor.fetchers.gateway_tokens import ensure_tokens_listed, token_addresses +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +MINT = "9QFfgxdSqH5zT7j6rZb1y6SZhw2aFtcQu2r6BuYpump" +USDC = "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v" +NETWORK = "solana-mainnet-beta" + + +class FakeGateway: + """Gateway's token list, with the two behaviours that matter kept honest.""" + + def __init__(self, listed=(), refuse_save=False, fail_search=False): + self.tokens = [{"address": a, "symbol": a[:4]} for a in listed] + self.refuse_save = refuse_save + self.fail_search = fail_search + self.searches: list[str] = [] + self.saves: list[str] = [] + self.deletes: list[str] = [] + + async def get_network_tokens(self, network_id, search=None): + if self.fail_search: + raise RuntimeError("gateway down") + self.searches.append(search) + # Gateway's `search` matches symbol and name only — never an address. + # Modelling it any other way is what hid the bug this fake now pins: + # a listed token searched for by mint came back empty and read unlisted. + needle = (search or "").lower() + return { + "tokens": [ + t + for t in self.tokens + if not needle + or needle in str(t.get("symbol") or "").lower() + or needle in str(t.get("name") or "").lower() + ] + } + + async def save_network_token(self, network_id, token_address): + self.saves.append(token_address) + # A ticker already held by another token: Gateway answers 200 and keeps + # the list unchanged. + if self.refuse_save: + return {"message": "already exists"} + self.tokens.append({"address": token_address, "symbol": "NEW"}) + return {"message": "added"} + + async def delete_token(self, network_id, token_address): + self.deletes.append(token_address) + before = len(self.tokens) + self.tokens = [t for t in self.tokens if t["address"] != token_address] + # The holder is deleted, so its ticker no longer blocks a save. + if len(self.tokens) < before: + self.refuse_save = False + return {"message": "deleted"} + + +class FakeClient: + def __init__(self, gateway): + self.gateway = gateway + + +@pytest.fixture(autouse=True) +def _clean_memo(): + gateway_tokens.reset_listed_memo() + yield + gateway_tokens.reset_listed_memo() + + +def test_a_missing_token_is_registered_without_being_asked_for(): + gw = FakeGateway(listed=[USDC]) + verdicts = asyncio.run(ensure_tokens_listed(FakeClient(gw), NETWORK, [MINT, USDC])) + + assert verdicts == {MINT: "added", USDC: "listed"} + assert gw.saves == [MINT] + + +def test_a_known_token_is_never_saved_twice(): + gw = FakeGateway(listed=[]) + client = FakeClient(gw) + + asyncio.run(ensure_tokens_listed(client, NETWORK, [MINT])) + second = asyncio.run(ensure_tokens_listed(client, NETWORK, [MINT])) + + assert second == {MINT: "listed"} + # One save, and no second lookup: the memo answers the repeat visit outright. + assert gw.saves == [MINT] + # The pre-check and the post-save verification, both reading the list whole + # (`search=None`) because Gateway cannot look a mint up by address. + assert gw.searches == [None, None] + + +def test_a_listed_token_is_not_asked_to_be_added_again(): + """The bug the pool banner showed: Solana USDC, listed, reported unlisted. + + Gateway's token search ignores addresses, so a lookup by mint answers + nothing. Reading that as "not listed" made the save that followed collide + with the token's *own* ticker ("already exists"), the verdict come back + `symbol_taken`, and the banner ask the user to add a token that was already + on the list — with no holder to name and no Replace that could help. + """ + gw = FakeGateway() + gw.tokens = [{"address": USDC, "symbol": "USDC", "name": "USDC"}] + + verdicts = asyncio.run(ensure_tokens_listed(FakeClient(gw), NETWORK, [USDC])) + + assert verdicts == {USDC: "listed"} + assert gw.saves == [] + + +def test_a_save_that_wrote_nothing_is_reported_not_assumed(): + # Gateway refuses a token whose ticker another token already holds — with a + # 200. Trusting the response would leave a token silently unlisted, and its + # balance silently 0. + gw = FakeGateway(listed=[], refuse_save=True) + verdicts = asyncio.run(ensure_tokens_listed(FakeClient(gw), NETWORK, [MINT])) + + assert verdicts == {MINT: "symbol_taken"} + + +def test_a_failed_lookup_stays_retryable(): + gw = FakeGateway(fail_search=True) + client = FakeClient(gw) + + assert asyncio.run(ensure_tokens_listed(client, NETWORK, [MINT])) == { + MINT: "failed" + } + + # Nothing was remembered, so a later visit tries again rather than trusting + # a state that was never confirmed. + gw.fail_search = False + assert asyncio.run(ensure_tokens_listed(client, NETWORK, [MINT])) == {MINT: "added"} + + +def test_only_address_shaped_values_are_sent_upstream(): + # A DEX pair is `-`: splitting it yields one thing + # that can be registered and one that cannot. + assert token_addresses(f"{MINT}-USDC") == [MINT] + assert token_addresses(MINT, MINT) == [MINT] # deduped + assert token_addresses("SOL-USDC", None, "") == [] + + +# --------------------------------------------------------------------------- +# The route the pool workspace calls on open. +# --------------------------------------------------------------------------- + +USER = WebUser(id=222, username="u", first_name="U", role="user") + + +class FakeConfigManager: + def __init__(self, gateway): + self.client = FakeClient(gateway) + + def has_server_access(self, user_id, name, *a, **kw): + return True + + async def get_client(self, name): + return self.client + + +@pytest.fixture +def route_client(monkeypatch): + gw = FakeGateway(listed=[USDC]) + cm = FakeConfigManager(gw) + monkeypatch.setattr(dex_routes, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + app = FastAPI() + app.include_router(dex_routes.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app), gw + + +def test_opening_a_pool_registers_both_of_its_tokens(route_client): + client, gw = route_client + + body = client.post( + "/servers/srv/dex/tokens", + json={"network": "solana-mainnet-beta", "addresses": [MINT, USDC]}, + ).json() + + assert body["tokens"] == {MINT: "added", USDC: "listed"} + assert gw.saves == [MINT] + + +def test_a_chain_named_the_gecko_way_still_reaches_gateway(route_client): + # Pool rows carry a chain id from whichever upstream produced them; Gateway + # only answers to its own spelling, so the route canonicalizes rather than + # rejecting a pool the user can plainly see. + client, _gw = route_client + + response = client.post( + "/servers/srv/dex/tokens", json={"network": "solana", "addresses": [MINT]} + ) + + assert response.status_code == 200 + assert response.json()["tokens"] == {MINT: "added"} + + +def test_a_chain_gateway_does_not_reach_is_refused(route_client): + client, _gw = route_client + + response = client.post( + "/servers/srv/dex/tokens", json={"network": "not-a-chain", "addresses": [MINT]} + ) + + assert response.status_code == 400 + + +def test_a_ticker_is_not_sent_upstream_as_an_address(route_client): + # `USDC` is a quote symbol, not something Gateway can look up by address. + client, gw = route_client + + body = client.post( + "/servers/srv/dex/tokens", + json={"network": "solana-mainnet-beta", "addresses": ["USDC", ""]}, + ).json() + + assert body["tokens"] == {} + assert gw.saves == [] + + +# --------------------------------------------------------------------------- +# The banner's add button: /dex/tokens/add, the human's answer to a verdict. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def make_client(monkeypatch): + def _make(gw): + cm = FakeConfigManager(gw) + monkeypatch.setattr(dex_routes, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + app = FastAPI() + app.include_router(dex_routes.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app) + + return _make + + +def test_the_add_button_registers_what_the_ensure_could_not(make_client): + client = make_client(FakeGateway(listed=[])) + + body = client.post( + "/servers/srv/dex/tokens/add", json={"network": NETWORK, "address": MINT} + ).json() + + assert body["verdict"] == "added" + + +def test_a_ticker_collision_names_its_holder(make_client): + # The verdict alone says the ticker is in use; the button's answer says by + # whom, which is what the user needs to decide whether to replace it. + gw = FakeGateway(refuse_save=True) + gw.tokens = [{"address": USDC, "symbol": "PUMP", "name": "Other Pump"}] + client = make_client(gw) + + body = client.post( + "/servers/srv/dex/tokens/add", + json={"network": NETWORK, "address": MINT, "symbol": "PUMP"}, + ).json() + + assert body["verdict"] == "symbol_taken" + assert body["conflict"]["address"] == USDC + assert gw.deletes == [] # naming the holder must not touch it + + +def test_replace_deletes_the_holder_then_registers(make_client): + gw = FakeGateway(refuse_save=True) + gw.tokens = [{"address": USDC, "symbol": "PUMP"}] + client = make_client(gw) + + body = client.post( + "/servers/srv/dex/tokens/add", + json={"network": NETWORK, "address": MINT, "symbol": "PUMP", "replace": True}, + ).json() + + assert gw.deletes == [USDC] + assert body["verdict"] == "added" + + +def test_a_replaced_holder_is_forgotten_by_the_memo(make_client): + # USDC was seen on the list and memoized; after a replace deletes it, a later + # ensure must re-read the list rather than trust a listing that is gone. + gw = FakeGateway(refuse_save=True) + gw.tokens = [{"address": USDC, "symbol": "PUMP"}] + client = make_client(gw) + assert asyncio.run(ensure_tokens_listed(FakeClient(gw), NETWORK, [USDC])) == { + USDC: "listed" + } + + client.post( + "/servers/srv/dex/tokens/add", + json={"network": NETWORK, "address": MINT, "symbol": "PUMP", "replace": True}, + ) + + verdicts = asyncio.run(ensure_tokens_listed(FakeClient(gw), NETWORK, [USDC])) + assert verdicts == {USDC: "added"} # re-registered, not answered from memory + + +def test_the_button_surfaces_a_gateway_error_instead_of_a_verdict(make_client): + # The automatic ensure folds errors into `failed`; a clicked button deserves + # the actual reason. + client = make_client(FakeGateway(fail_search=True)) + + response = client.post( + "/servers/srv/dex/tokens/add", json={"network": NETWORK, "address": MINT} + ) + + assert response.status_code == 502 + + +def test_the_button_refuses_a_value_that_is_not_an_address(make_client): + client = make_client(FakeGateway()) + + response = client.post( + "/servers/srv/dex/tokens/add", json={"network": NETWORK, "address": "USDC"} + ) + + assert response.status_code == 400 diff --git a/tests/test_local_mode.py b/tests/test_local_mode.py new file mode 100644 index 000000000..ed1ffefb4 --- /dev/null +++ b/tests/test_local_mode.py @@ -0,0 +1,433 @@ +"""Condor runs without Telegram (FEAT-049). + +Two of these tests are the feature's safety controls and exist to fail loudly: + +* **Mode is never inferred.** ``CONDOR_MODE`` unset with an empty + ``TELEGRAM_TOKEN`` is a hard error naming ``make setup`` — an install that + loses its token must stop, never quietly become an unauthenticated dashboard. +* **Local mode binds loopback only.** ``WEB_HOST`` is the one, explicit, + documented way to expose a dashboard that has no login at all. + +Everything else pins the seams: the local-login endpoint exists only in local +mode, the JWT it mints is an ordinary one (``get_current_user`` never learns +which mode it is in), the identity ``make setup`` writes actually resolves, and +PTB still runs a job queue on an Application that was never initialized. + +The identity tests carry a second lesson. Local mode logs in as +``ADMIN_USER_ID`` and nothing else: a dedicated ``CONDOR_LOCAL_USER_ID`` was a +second knob for one identity, and the two disagreed the moment a Telegram +install flipped ``CONDOR_MODE=local``. Whether that identity resolves is now +settled at boot, where the .env is, rather than by a 500 from the browser. + +Sync tests driving coroutines with ``asyncio.run``: ``pytest-asyncio`` is a dev +dependency but is not installed in this venv (same convention as +``tests/test_notifications.py``). +""" + +import asyncio +import ipaddress + +import pytest +from starlette.testclient import TestClient + +from condor.web.app import create_app +from config_manager import ConfigManager, ServerPermission, UserRole +from utils import config as app_config +from utils.config import ( + ConfigError, + check_local_user, + check_startup_config, + resolve_admin_id, + resolve_local_user_id, + resolve_mode, + resolve_web_host, +) + +LOCAL_ID = 1 + + +# ── Mode resolution: explicit, never inferred ── + + +@pytest.mark.parametrize( + "env, expected", + [ + ({}, "telegram"), + ({"CONDOR_MODE": ""}, "telegram"), + ({"CONDOR_MODE": " "}, "telegram"), + ({"CONDOR_MODE": "telegram"}, "telegram"), + ({"CONDOR_MODE": "local"}, "local"), + ({"CONDOR_MODE": "LOCAL"}, "local"), + ({"CONDOR_MODE": " Local "}, "local"), + # A typo must not land in local mode. Telegram is the fail-safe answer: + # it then demands a token, which is the hard error below. + ({"CONDOR_MODE": "locl"}, "telegram"), + # The presence of a token says nothing about the mode, in either + # direction: this is the inference that must never happen. + ({"TELEGRAM_TOKEN": ""}, "telegram"), + ({"CONDOR_MODE": "local", "TELEGRAM_TOKEN": "123:abc"}, "local"), + ], +) +def test_resolve_mode(env, expected): + assert resolve_mode(env) == expected + + +def test_no_mode_and_no_token_is_a_hard_error_naming_setup(): + """CONTROL (a). The failure this feature must never have. + + An install whose token went missing has to stop with an instruction, not + fall through to a dashboard anybody can walk into. + """ + with pytest.raises(ConfigError) as excinfo: + check_startup_config({}) + + message = str(excinfo.value) + assert "make setup" in message + assert "CONDOR_MODE" in message + + +@pytest.mark.parametrize( + "env", + [ + {}, + {"TELEGRAM_TOKEN": ""}, + {"TELEGRAM_TOKEN": " "}, + {"CONDOR_MODE": "telegram"}, + {"CONDOR_MODE": "telegram", "TELEGRAM_TOKEN": ""}, + # A mistyped mode is telegram, so it still needs a token. + {"CONDOR_MODE": "locall", "TELEGRAM_TOKEN": ""}, + ], +) +def test_telegram_mode_without_a_token_refuses_to_start(env): + """CONTROL (a), across the whole env matrix that could mean 'no token'.""" + with pytest.raises(ConfigError): + check_startup_config(env) + + +@pytest.mark.parametrize( + "env", + [ + {"TELEGRAM_TOKEN": "123:abc"}, + {"CONDOR_MODE": "telegram", "TELEGRAM_TOKEN": "123:abc"}, + # Local mode is the only way to legitimately run with no token. + {"CONDOR_MODE": "local"}, + {"CONDOR_MODE": "local", "TELEGRAM_TOKEN": ""}, + ], +) +def test_valid_configurations_start(env): + check_startup_config(env) # must not raise + + +# ── Bind address ── + + +def test_local_mode_binds_loopback_only(): + """CONTROL (b). Local mode has no authentication; it must not be routable.""" + host = resolve_web_host({"CONDOR_MODE": "local"}) + + assert ipaddress.ip_address(host).is_loopback + assert host == "127.0.0.1" + + +def test_telegram_mode_still_binds_all_interfaces(): + """Telegram mode authenticates, and its binding is unchanged.""" + assert resolve_web_host({}) == "0.0.0.0" + assert resolve_web_host({"CONDOR_MODE": "telegram"}) == "0.0.0.0" + + +@pytest.mark.parametrize("mode", ["local", "telegram", ""]) +def test_web_host_is_the_explicit_opt_out(mode): + """CONTROL (b)'s escape hatch: exposing it has to be a deliberate act.""" + env = {"CONDOR_MODE": mode, "WEB_HOST": "0.0.0.0"} + assert resolve_web_host(env) == "0.0.0.0" + + env["WEB_HOST"] = "10.0.0.5" + assert resolve_web_host(env) == "10.0.0.5" + + +def test_uvicorn_binds_the_resolved_host(monkeypatch): + """CONTROL (b) is wired, not just computed: uvicorn gets WEB_HOST.""" + import main + + monkeypatch.setattr(main, "WEB_HOST", "127.0.0.1") + assert ipaddress.ip_address(main._web_server_config(object()).host).is_loopback + + monkeypatch.setattr(main, "WEB_HOST", "0.0.0.0") + assert main._web_server_config(object()).host == "0.0.0.0" + + +# ── Local identity ── + + +@pytest.mark.parametrize( + "env, expected", + [ + ({}, 1), + ({"ADMIN_USER_ID": ""}, 1), + ({"ADMIN_USER_ID": " "}, 1), + # The admin *is* the local user. There is no second knob to disagree + # with this one. + ({"ADMIN_USER_ID": "42"}, 42), + ({"ADMIN_USER_ID": " 123456789 "}, 123456789), + # Junk and falsy ids fall back to 1: several call sites read a falsy + # user or chat id as "absent", so 0 would silently address nobody. + # check_startup_config refuses to boot on these separately. + ({"ADMIN_USER_ID": "nope"}, 1), + ({"ADMIN_USER_ID": "0"}, 1), + ({"ADMIN_USER_ID": "-5"}, 1), + ], +) +def test_resolve_local_user_id(env, expected): + assert resolve_local_user_id(env) == expected + + +def test_local_user_is_the_admin_when_a_telegram_install_flips_mode(): + """The regression this collapse exists for. + + A Telegram install that sets ``CONDOR_MODE=local`` keeps its Telegram + ``ADMIN_USER_ID``. Local mode used to log in as a hardcoded ``1`` that + ``config.yml`` had never heard of, so ``/auth/local-login`` 500'd on an + install that was, by every other measure, correctly configured. + """ + env = {"CONDOR_MODE": "local", "ADMIN_USER_ID": "123456789"} + + assert resolve_local_user_id(env) == 123456789 + assert resolve_local_user_id(env) == resolve_admin_id(env) + + +@pytest.mark.parametrize("raw", ["ralphsohandsome", "1.5", "0", "-5", "1,2"]) +def test_an_unusable_admin_id_refuses_to_start(raw): + """A typo'd ADMIN_USER_ID used to be swallowed, leaving *no admin at all*. + + No admin panel, no approvals, no boot notification, and in local mode no + user to log in as — each of which surfaces later as unrelated breakage. + """ + with pytest.raises(ConfigError) as excinfo: + check_startup_config({"ADMIN_USER_ID": raw, "TELEGRAM_TOKEN": "123:abc"}) + + assert "ADMIN_USER_ID" in str(excinfo.value) + assert raw in str(excinfo.value) + + +@pytest.mark.parametrize("raw", ["1", "123456789", " 42 "]) +def test_a_usable_admin_id_starts(raw): + check_startup_config({"ADMIN_USER_ID": raw, "TELEGRAM_TOKEN": "123:abc"}) + + +# ── Local mode's user is checked at boot, not at login ── + + +def test_local_mode_refuses_to_start_on_an_unconfigured_user(): + """Ralph's report: the failure has to land in ``make run``, not the browser. + + A dashboard that boots, opens, auto-logs-in and *then* 500s tells you + nothing about the .env line that caused it. + """ + with pytest.raises(ConfigError) as excinfo: + check_local_user({"CONDOR_MODE": "local"}, get_role=lambda _uid: None) + + message = str(excinfo.value) + assert "ADMIN_USER_ID" in message + assert "make setup" in message + + +def test_local_mode_names_the_admin_id_it_actually_tried(): + """The message has to name the id, or it sends you looking in config.yml.""" + env = {"CONDOR_MODE": "local", "ADMIN_USER_ID": "999"} + + with pytest.raises(ConfigError) as excinfo: + check_local_user(env, get_role=lambda _uid: None) + + assert "999" in str(excinfo.value) + + +@pytest.mark.parametrize("role", [UserRole.ADMIN, UserRole.USER]) +def test_local_mode_starts_when_the_user_resolves(role): + check_local_user({"CONDOR_MODE": "local"}, get_role=lambda _uid: role) + + +@pytest.mark.parametrize("role", [UserRole.PENDING, UserRole.BLOCKED, None]) +def test_local_mode_refuses_a_user_who_cannot_use_the_dashboard(role): + """Same bar as ``get_current_user``: pending and blocked are not sessions.""" + with pytest.raises(ConfigError): + check_local_user({"CONDOR_MODE": "local"}, get_role=lambda _uid: role) + + +@pytest.mark.parametrize("env", [{}, {"CONDOR_MODE": "telegram"}, {"CONDOR_MODE": ""}]) +def test_the_local_user_check_is_a_no_op_in_telegram_mode(env): + """Telegram mode has its own auth; this check must never touch it.""" + + def _explode(_uid): # pragma: no cover - must not be called + raise AssertionError("config.yml consulted outside local mode") + + check_local_user(env, get_role=_explode) + + +def test_setup_template_gives_the_local_user_a_working_identity(tmp_path, monkeypatch): + """What ``make setup`` writes in local mode has to actually resolve. + + ``ADMIN_USER_ID=1`` in ``.env`` plus ``admin_id: 1`` and + ``server_access.local.owner_id: 1`` in ``config.yml`` is the whole identity + story — no new concept, just the integer everything already keys on. If this + breaks, ``/auth/local-login`` 500s and nothing else works. + + The ``.env`` half is load-bearing, not decoration: ``ConfigManager`` only + materialises the admin's *user record* from the environment's + ``ADMIN_USER_ID`` (``_get_admin_from_env``), so a ``config.yml`` naming + ``admin_id: 1`` with an empty ``users:`` grants nothing on its own. + """ + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_config, "ADMIN_USER_ID", LOCAL_ID) + (tmp_path / "config.yml").write_text( + """servers: + local: + host: localhost + port: 8000 + username: admin + password: admin + +default_server: local + +admin_id: 1 + +users: {} + +server_access: + local: + owner_id: 1 + created_at: null + shared_with: {} + +chat_defaults: + 1: local + +version: 1 +""", + encoding="utf-8", + ) + + cm = ConfigManager(config_path=str(tmp_path / "config.yml")) + + assert cm.get_user_role(LOCAL_ID) == UserRole.ADMIN + assert cm.has_server_access(LOCAL_ID, "local", ServerPermission.TRADER) + + +# ── The one mode-aware endpoint ── + + +@pytest.fixture +def client(monkeypatch): + """A dashboard whose config manager knows exactly one user: the local admin.""" + monkeypatch.setenv("WEB_JWT_SECRET", "test-secret-for-local-mode") + + class _CM: + def get_user_role(self, user_id): + return UserRole.ADMIN if user_id == LOCAL_ID else None + + monkeypatch.setattr("condor.web.routes.auth.get_config_manager", lambda: _CM()) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: _CM()) + monkeypatch.setattr(app_config, "LOCAL_USER_ID", LOCAL_ID) + with TestClient(create_app()) as test_client: + yield test_client + + +def _set_mode(monkeypatch, mode: str) -> None: + monkeypatch.setattr(app_config, "CONDOR_MODE", mode) + monkeypatch.setattr(app_config, "LOCAL_MODE", mode == "local") + + +def test_local_login_does_not_exist_in_telegram_mode(client, monkeypatch): + """CONTROL, second half: no passwordless login exists where anyone can reach it.""" + _set_mode(monkeypatch, "telegram") + assert client.post("/api/v1/auth/local-login").status_code == 404 + + +def test_local_login_does_not_exist_when_mode_is_unset(client, monkeypatch): + """An install that never heard of CONDOR_MODE has no local login.""" + _set_mode(monkeypatch, resolve_mode({})) + assert client.post("/api/v1/auth/local-login").status_code == 404 + + +def test_local_login_mints_a_session_the_shared_guard_accepts(client, monkeypatch): + """The JWT is an ordinary one: ``get_current_user`` never learns the mode.""" + _set_mode(monkeypatch, "local") + + res = client.post("/api/v1/auth/local-login") + assert res.status_code == 200 + body = res.json() + assert body["user"]["id"] == LOCAL_ID + assert body["user"]["role"] == "admin" + assert res.headers["Referrer-Policy"] == "no-referrer" + assert res.headers["Cache-Control"] == "no-store" + + # The unchanged guard, reached through an unchanged endpoint. + me = client.get( + "/api/v1/auth/me", headers={"Authorization": f"Bearer {body['token']}"} + ) + assert me.status_code == 200 + assert me.json()["id"] == LOCAL_ID + + +def test_local_login_refuses_an_unconfigured_local_user(client, monkeypatch): + """No config.yml entry means no session — not a session for a stranger.""" + _set_mode(monkeypatch, "local") + monkeypatch.setattr(app_config, "LOCAL_USER_ID", 4242) + + res = client.post("/api/v1/auth/local-login") + assert res.status_code == 500 + assert "make setup" in res.json()["detail"] + + +def test_auth_mode_reports_the_mode_without_a_session(client, monkeypatch): + """The Login page has to know how to log in before it can.""" + _set_mode(monkeypatch, "local") + assert client.get("/api/v1/auth/mode").json() == {"mode": "local"} + + _set_mode(monkeypatch, "telegram") + assert client.get("/api/v1/auth/mode").json() == {"mode": "telegram"} + + +# ── Boot without Telegram ── + + +def test_job_queue_runs_on_an_uninitialized_application(): + """Local mode's whole boot rests on this PTB behaviour, so pin it. + + Scheduled routines, update checks and signals are all ``job_queue`` jobs. + Local mode never calls ``initialize()``/``start()`` (nothing polls), it just + starts the queue — a PTB release that breaks that must fail here rather than + in someone's install. + """ + from telegram.ext import Application + + async def scenario(): + application = Application.builder().token("0:local").build() + fired = asyncio.Event() + + async def _job(_context): + fired.set() + + # Scheduled *before* the queue starts, exactly as ``startup()`` does. + application.job_queue.run_once(_job, 0) + await application.job_queue.start() + try: + await asyncio.wait_for(fired.wait(), timeout=5) + finally: + await application.job_queue.stop() + + asyncio.run(scenario()) + + +def test_outbound_messages_go_to_the_bell_in_local_mode(monkeypatch): + """``context.bot`` is never None: with no Telegram it is the dashboard bell.""" + import main + from condor.notifications import NotifyBot + + class _App: + bot = object() + + monkeypatch.setattr(main, "LOCAL_MODE", True) + assert isinstance(main._outbound_bot(_App()), NotifyBot) + + monkeypatch.setattr(main, "LOCAL_MODE", False) + assert main._outbound_bot(_App()) is _App.bot diff --git a/tests/test_market_data_fetchers_strict.py b/tests/test_market_data_fetchers_strict.py new file mode 100644 index 000000000..c87f34295 --- /dev/null +++ b/tests/test_market_data_fetchers_strict.py @@ -0,0 +1,82 @@ +"""Tests for CORR-188: ticker/price fetchers expose a strict mode for the SDS. + +``fetch_current_price``, ``fetch_tickers`` and ``fetch_ticker_pool`` are cached +reads in the ServerDataService, whose invariant is that a failed fetch must +*raise* so ``_do_fetch_and_cache`` keeps the last good value, records the error +and backs off — instead of caching an empty shape as a fresh success. These +tests pin the ``strict=True`` re-raise, the 404 "endpoint absent" carve-out +(a real answer, never a failure), and the lenient default for direct callers. +""" + +import asyncio + +import pytest + +from condor.fetchers.market_data import ( + fetch_current_price, + fetch_ticker_pool, + fetch_tickers, +) + +EMPTY_POOL = {"connectors": {}, "prices": {}, "updated_at": {}} +EMPTY_TICKERS = {"tickers": {}, "updated_at": None} + + +class FailingClient: + """Client whose market-data endpoints raise a given exception.""" + + def __init__(self, exc: Exception): + self.market_data = self._MarketData(exc) + + class _MarketData: + def __init__(self, exc): + self._exc = exc + + async def get_tickers(self, connector_name=None): + raise self._exc + + async def get_prices(self, connector_name="", trading_pairs=""): + raise self._exc + + +def test_ticker_pool_strict_reraises_transient_errors(): + client = FailingClient(RuntimeError("500 upstream exploded")) + with pytest.raises(RuntimeError): + asyncio.run(fetch_ticker_pool(client, strict=True)) + + +def test_ticker_pool_404_is_a_real_empty_answer_even_in_strict_mode(): + client = FailingClient(RuntimeError("404 Not Found")) + assert asyncio.run(fetch_ticker_pool(client, strict=True)) == EMPTY_POOL + + +def test_ticker_pool_lenient_default_degrades_to_empty(): + client = FailingClient(RuntimeError("500 upstream exploded")) + assert asyncio.run(fetch_ticker_pool(client)) == EMPTY_POOL + + +def test_tickers_strict_reraises_transient_errors(): + client = FailingClient(TimeoutError("timed out")) + with pytest.raises(TimeoutError): + asyncio.run(fetch_tickers(client, "binance", strict=True)) + + +def test_tickers_404_is_a_real_empty_answer_even_in_strict_mode(): + client = FailingClient(RuntimeError("404 Not Found")) + assert asyncio.run(fetch_tickers(client, "binance", strict=True)) == EMPTY_TICKERS + + +def test_tickers_lenient_default_degrades_to_empty(): + client = FailingClient(RuntimeError("500 upstream exploded")) + assert asyncio.run(fetch_tickers(client, "binance")) == EMPTY_TICKERS + + +def test_current_price_strict_reraises(): + client = FailingClient(RuntimeError("connection refused")) + with pytest.raises(RuntimeError): + asyncio.run(fetch_current_price(client, "binance", "BTC-USDT", strict=True)) + + +def test_current_price_lenient_default_degrades_to_none(): + client = FailingClient(RuntimeError("connection refused")) + assert asyncio.run(fetch_current_price(client, "binance", "BTC-USDT")) is None diff --git a/tests/test_memory_store.py b/tests/test_memory_store.py index ef9f38c08..421b06cc6 100644 --- a/tests/test_memory_store.py +++ b/tests/test_memory_store.py @@ -218,7 +218,7 @@ def test_concurrent_writers_never_leave_a_torn_file(memory_root): # always parse cleanly (frontmatter + body), never a torn/interleaved one. import threading - from condor.memory.store import _parse_frontmatter + from condor.frontmatter import parse_frontmatter s = MemoryStore(user_id=42) target = s.memories_dir / "shared.md" @@ -240,7 +240,7 @@ def writer(text): for t in threads: t.join() - meta, body = _parse_frontmatter(target.read_text()) + meta, body = parse_frontmatter(target.read_text()) # The surviving file is exactly one writer's payload, not a blend. assert meta.get("name") == "shared" assert body == "x" * 5000 diff --git a/tests/test_notifications.py b/tests/test_notifications.py new file mode 100644 index 000000000..474b8e02e --- /dev/null +++ b/tests/test_notifications.py @@ -0,0 +1,421 @@ +"""Task notifications reach the dashboard (FEAT-048). + +What is pinned here is the promise the feature makes: a finished background task +is *addressed to a user*, survives nobody watching, never crosses to another +user, and never costs its producer anything when the surface is down. + +Sync tests driving coroutines with ``asyncio.run``: ``pytest-asyncio`` is a dev +dependency but is not installed in this venv (same convention as +``test_notify_route.py``). +""" + +import asyncio +import json + +import pytest + +from condor import notifications +from condor.notifications import ( + NotifyBot, + list_for, + mark_read, + record, + unread_count, + user_for_chat, +) +from condor.web.models import WebUser + +USER_A = 111 +USER_B = 222 + + +@pytest.fixture +def store(tmp_path, monkeypatch): + """A private store file and an empty sink registry for each test.""" + path = tmp_path / "notifications.json" + monkeypatch.setattr(notifications, "_FILE", path) + monkeypatch.setattr(notifications, "_push_sinks", []) + return path + + +@pytest.fixture +def known_users(monkeypatch): + """Both test users exist as far as ``config_manager`` is concerned.""" + + class _CM: + def get_user(self, user_id): + return {"id": user_id} if user_id in (USER_A, USER_B) else None + + monkeypatch.setattr("config_manager.get_config_manager", lambda: _CM()) + + +# ── The store ── + + +def test_record_persists_and_reloads(store): + """A notification outlives the process that produced it.""" + n = asyncio.run(record(USER_A, "task done", kind="delegation")) + + assert n is not None + assert store.exists() + on_disk = json.loads(store.read_text()) + assert list(on_disk) == [str(USER_A)] + + items = list_for(USER_A) + assert [i.text for i in items] == ["task done"] + assert items[0].kind == "delegation" + assert items[0].read is False + assert unread_count(USER_A) == 1 + + +def test_newest_first_and_capped(store, monkeypatch): + """The bell keeps the recent past, not all of it.""" + monkeypatch.setattr(notifications, "_MAX_PER_USER", 5) + + async def fill(): + for i in range(8): + await record(USER_A, f"note {i}") + + asyncio.run(fill()) + + texts = [n.text for n in list_for(USER_A, limit=100)] + assert texts == ["note 7", "note 6", "note 5", "note 4", "note 3"] + + +def test_users_never_see_each_other(store): + """The store is per user, and reading is scoped to the reader.""" + + async def both(): + await record(USER_A, "A's task") + await record(USER_B, "B's task") + + asyncio.run(both()) + + assert [n.text for n in list_for(USER_A)] == ["A's task"] + assert [n.text for n in list_for(USER_B)] == ["B's task"] + + +def test_mark_read_cannot_reach_across_users(store): + """An id from someone else's bucket is simply not found.""" + b = asyncio.run(record(USER_B, "B's task")) + assert b is not None + + left = mark_read(USER_A, [b.id]) + + assert left == 0 # A has nothing of their own + assert unread_count(USER_B) == 1 + assert list_for(USER_B)[0].read is False + + +def test_mark_read_all_and_by_id(store): + async def three(): + return [await record(USER_A, f"note {i}") for i in range(3)] + + made = asyncio.run(three()) + + assert mark_read(USER_A, [made[0].id]) == 2 + assert mark_read(USER_A, None) == 0 + assert unread_count(USER_A) == 0 + + +def test_no_user_records_nothing(store): + """A producer with nobody behind it is not an error.""" + assert asyncio.run(record(0, "orphan")) is None + assert asyncio.run(record(None, "orphan")) is None + assert asyncio.run(record(USER_A, "")) is None + assert not store.exists() + + +def test_concurrent_records_do_not_lose_each_other(store): + """The write lock is what keeps a simultaneous finish from vanishing.""" + + async def both(): + await asyncio.gather( + *[record(USER_A, f"note {i}") for i in range(10)], + ) + + asyncio.run(both()) + assert len(list_for(USER_A, limit=100)) == 10 + + +# ── The push bus ── + + +def test_push_reaches_registered_sink(store): + seen: list = [] + + async def sink(n): + seen.append(n) + + notifications.register_push_sink(sink) + asyncio.run(record(USER_A, "live")) + + assert [n.text for n in seen] == ["live"] + + +def test_dead_sink_never_reaches_the_producer(store): + """A socket that blew up must not fail the task that was announcing itself.""" + + async def broken(n): + raise RuntimeError("socket is gone") + + notifications.register_push_sink(broken) + + n = asyncio.run(record(USER_A, "still recorded")) + + assert n is not None + assert [i.text for i in list_for(USER_A)] == ["still recorded"] + + +# ── The WS surface ── + + +class _FakeWS: + def __init__(self, explode=False): + self.sent: list[dict] = [] + self.explode = explode + + async def send_text(self, raw: str) -> None: + if self.explode: + raise RuntimeError("closed") + self.sent.append(json.loads(raw)) + + +def test_ws_push_reaches_only_the_addressed_user(store, monkeypatch): + from condor.web.routes import chat_ws + + monkeypatch.setattr(notifications, "_push_sinks", [chat_ws._push_notification]) + a_ws, b_ws = _FakeWS(), _FakeWS() + monkeypatch.setattr(chat_ws, "_attached_sockets", {USER_A: {a_ws}, USER_B: {b_ws}}) + + asyncio.run(record(USER_A, "A's task", kind="delegation", link="/routines")) + + assert b_ws.sent == [] + assert len(a_ws.sent) == 1 + frame = a_ws.sent[0] + assert frame["event"] == "notification" + assert frame["kind"] == "delegation" + assert frame["text"] == "A's task" + assert frame["link"] == "/routines" + assert frame["id"] and frame["ts"] + + +def test_ws_push_on_a_dead_socket_still_stores(store, monkeypatch): + from condor.web.routes import chat_ws + + monkeypatch.setattr(notifications, "_push_sinks", [chat_ws._push_notification]) + monkeypatch.setattr(chat_ws, "_attached_sockets", {USER_A: {_FakeWS(explode=True)}}) + + asyncio.run(record(USER_A, "unwatched")) + + assert [n.text for n in list_for(USER_A)] == ["unwatched"] + + +# ── The REST surface ── + + +def test_rest_is_scoped_to_the_jwt(store): + from condor.web.routes.notifications import ( + MarkReadRequest, + list_notifications, + mark_notifications_read, + ) + + async def both(): + await record(USER_A, "A's task") + b = await record(USER_B, "B's task") + return b + + b = asyncio.run(both()) + + listing = asyncio.run( + list_notifications(limit=50, user=WebUser(id=USER_A, role="user")) + ) + assert [i["text"] for i in listing["items"]] == ["A's task"] + assert listing["unread"] == 1 + + # A's session asking to read B's id changes nothing about B. + asyncio.run( + mark_notifications_read( + MarkReadRequest(ids=[b.id]), user=WebUser(id=USER_A, role="user") + ) + ) + assert unread_count(USER_B) == 1 + + +# ── The adapter and the outbound ladder ── + + +def test_notify_bot_records_a_send(store, known_users): + bot = NotifyBot() + + asyncio.run(bot.send_message(chat_id=USER_A, text="hello")) + + assert [n.text for n in list_for(USER_A)] == ["hello"] + + +def test_notify_bot_ignores_a_chat_with_no_owner(store, known_users): + bot = NotifyBot() + + asyncio.run(bot.send_message(chat_id=-100123, text="group chatter")) + asyncio.run(bot.send_message(chat_id=999999, text="stranger")) + + assert user_for_chat(-100123) is None + assert list_for(999999) == [] + assert not store.exists() + + +def test_notify_bot_documents_are_linked_not_rehosted(store, known_users): + bot = NotifyBot() + + asyncio.run(bot.send_document(chat_id=USER_A, document=b"x", filename="Daily.html")) + + item = list_for(USER_A)[0] + assert item.text == "Daily.html" + assert item.link == "/routines?tab=reports" + + +def test_resolve_bot_falls_through_to_the_bell_without_a_token( + store, known_users, monkeypatch +): + """With no Telegram at all, an outbound message becomes a bell item.""" + monkeypatch.delenv("TELEGRAM_TOKEN", raising=False) + monkeypatch.delenv("TELEGRAM_BOT_TOKEN", raising=False) + + class _NoBotStore: + def get_bot(self): + return None + + monkeypatch.setattr("condor.routine_store.get_routine_store", lambda: _NoBotStore()) + + from condor.agents.delegate import resolve_bot + + bot = resolve_bot() + assert isinstance(bot, NotifyBot) + + asyncio.run(bot.send_message(chat_id=USER_A, text="dropped no more")) + assert [n.text for n in list_for(USER_A)] == ["dropped no more"] + + +def test_resolve_bot_prefers_telegram_when_there_is_a_token(monkeypatch): + """A configured install is byte-for-byte unchanged.""" + monkeypatch.setenv("TELEGRAM_TOKEN", "123:abc") + + class _NoBotStore: + def get_bot(self): + return None + + monkeypatch.setattr("condor.routine_store.get_routine_store", lambda: _NoBotStore()) + + from condor.agents.delegate import resolve_bot + from condor.routine_store import _HttpBot + + assert isinstance(resolve_bot(), _HttpBot) + + +# ── Producers ── + + +def test_delegation_completion_lights_the_bell_once(store, known_users): + """One item, with exactly the text Telegram received.""" + from condor.agents.delegate import DelegateTask, _completion_text, _notify_done + + dt = DelegateTask( + task_id="t1", + agent_slug="condor", + user_id=USER_A, + chat_id=USER_A, + server_name=None, + task="do the thing", + status="done", + result="all done", + ) + + sent: list[dict] = [] + + class _Bot: + async def send_message(self, **kw): + sent.append(kw) + return {"ok": True} + + asyncio.run(_notify_done(dt, _Bot())) + + assert len(sent) == 1 + items = list_for(USER_A) + assert len(items) == 1 + assert items[0].text == _completion_text(dt) == sent[0]["text"] + assert items[0].kind == "delegation" + + +def test_delegation_over_the_bell_ladder_is_not_recorded_twice( + store, known_users, monkeypatch +): + """When the bell *is* the sender, ``_notify_done`` must not file it again.""" + monkeypatch.delenv("TELEGRAM_TOKEN", raising=False) + monkeypatch.delenv("TELEGRAM_BOT_TOKEN", raising=False) + + class _NoBotStore: + def get_bot(self): + return None + + monkeypatch.setattr("condor.routine_store.get_routine_store", lambda: _NoBotStore()) + + from condor.agents.delegate import DelegateTask, _notify_done + + dt = DelegateTask( + task_id="t2", + agent_slug="condor", + user_id=USER_A, + chat_id=USER_A, + server_name=None, + task="do the thing", + status="done", + result="all done", + ) + + asyncio.run(_notify_done(dt, None)) + + assert len(list_for(USER_A)) == 1 + + +def test_finished_routine_links_to_the_reports_tab(store): + """Even with no conversation behind the run, the owner is told.""" + from condor.routine_store import RoutineStore + + rs = RoutineStore() + rs._instances["i1"] = {"routine_name": "daily_pnl", "user_id": USER_A} + + asyncio.run(rs._report_run("i1", "12 pools checked", None)) + + item = list_for(USER_A)[0] + assert item.kind == "routine" + assert item.link == "/routines?tab=reports" + assert "daily_pnl" in item.text + + +def test_notify_route_counts_a_bell_only_delivery_as_recorded(store, monkeypatch): + """MCP ``send_notification`` succeeds on an install with no Telegram.""" + from condor.web.routes import agents as agents_routes + from condor.web.routes.agents import NotifyRequest, notify_user + + async def _no_conversation(session_key: str) -> str: + return "" + + monkeypatch.setattr(agents_routes, "_conversation_for_session", _no_conversation) + + class _DeadBot: + async def send_message(self, **kw): + return None + + monkeypatch.setattr("condor.agents.delegate.resolve_bot", lambda b=None: _DeadBot()) + + result = asyncio.run( + notify_user( + NotifyRequest(text="the thing happened", chat_id=USER_A), + user=WebUser(id=USER_A, role="user"), + ) + ) + + assert result["sent"] is False + assert result["recorded"] is True + assert [n.text for n in list_for(USER_A)] == ["the thing happened"] diff --git a/tests/test_notify_route.py b/tests/test_notify_route.py index 470e31435..6e6566cb2 100644 --- a/tests/test_notify_route.py +++ b/tests/test_notify_route.py @@ -20,6 +20,8 @@ from condor.web.routes.agents import NotifyRequest, notify_user from mcp_servers.condor.tools import notification as notification_tool +# The pushes below target chat 1 — the caller's own private chat. A foreign +# chat id is refused since SEC-198 (see test_agents_chat_id_ownership.py). CALLER = WebUser(id=1, role="user") @@ -86,7 +88,7 @@ def test_a_live_session_gets_both_a_transcript_note_and_the_telegram_push( notify_user( NotifyRequest( text="LP position closed", - chat_id=42, + chat_id=1, session_key="web:1:slot-1", ), user=CALLER, @@ -96,7 +98,7 @@ def test_a_live_session_gets_both_a_transcript_note_and_the_telegram_push( assert result == {"sent": True, "recorded": True} assert notes == [(1, "conv-1", "LP position closed", "notification")] assert len(bot.calls) == 1 - assert bot.calls[0]["chat_id"] == 42 + assert bot.calls[0]["chat_id"] == 1 def test_the_note_is_scoped_to_the_jwt_caller_not_the_posted_user_id( @@ -107,7 +109,7 @@ def test_the_note_is_scoped_to_the_jwt_caller_not_the_posted_user_id( asyncio.run( notify_user( - NotifyRequest(text="hi", chat_id=42, user_id=999, session_key="web:1:s"), + NotifyRequest(text="hi", chat_id=1, user_id=999, session_key="web:1:s"), user=CALLER, ) ) @@ -119,10 +121,13 @@ def test_a_dead_session_key_still_pushes_and_notes_nothing(monkeypatch, bot, not _resolves_to(monkeypatch, "") result = asyncio.run( - notify_user(NotifyRequest(text="ping", chat_id=42, session_key=""), user=CALLER) + notify_user(NotifyRequest(text="ping", chat_id=1, session_key=""), user=CALLER) ) - assert result == {"sent": True, "recorded": False} + # No conversation to write into, so no transcript note -- but the dashboard + # bell is addressed to the user, not to a conversation, so it still records + # and the call still counts as delivered (FEAT-048). + assert result == {"sent": True, "recorded": True} assert notes == [] assert len(bot.calls) == 1 @@ -140,11 +145,13 @@ def explode(*a, **kw): result = asyncio.run( notify_user( - NotifyRequest(text="ping", chat_id=42, session_key="web:1:s"), user=CALLER + NotifyRequest(text="ping", chat_id=1, session_key="web:1:s"), user=CALLER ) ) - assert result == {"sent": True, "recorded": False} + # ``recorded`` is now true on the bell alone (FEAT-048): the transcript + # write failed, and that must not cost the user either delivery. + assert result == {"sent": True, "recorded": True} assert len(bot.calls) == 1 @@ -156,7 +163,7 @@ def test_bad_markdown_is_retried_as_plain_text(monkeypatch, notes): monkeypatch.setattr(delegate_module, "resolve_bot", lambda b=None: fake) result = asyncio.run( - notify_user(NotifyRequest(text="a_b_c", chat_id=42), user=CALLER) + notify_user(NotifyRequest(text="a_b_c", chat_id=1), user=CALLER) ) assert result["sent"] is True @@ -181,7 +188,7 @@ def test_empty_text_is_rejected(monkeypatch, bot, notes): _resolves_to(monkeypatch, "conv-1") with pytest.raises(HTTPException) as exc: - asyncio.run(notify_user(NotifyRequest(text="", chat_id=42), user=CALLER)) + asyncio.run(notify_user(NotifyRequest(text="", chat_id=1), user=CALLER)) assert exc.value.status_code == 400 diff --git a/tests/test_ohlcv_upstream_coalescing.py b/tests/test_ohlcv_upstream_coalescing.py index cf53274a7..203cb7113 100644 --- a/tests/test_ohlcv_upstream_coalescing.py +++ b/tests/test_ohlcv_upstream_coalescing.py @@ -12,7 +12,7 @@ import pytest import condor.web.routes.market as market -from condor import dex_candles, pool_data +from condor import pool_data from condor.web.models import WebUser POOL = "So11111111111111111111111111111111111111112" @@ -24,7 +24,7 @@ def _clean_state(): def _reset(): market._candle_cache.clear() market._candle_inflight.clear() - dex_candles._ohlcv_cache.clear() + # Also clears pool_data's process-wide OHLCV cache. pool_data.reset_gecko_throttle() _reset() @@ -105,23 +105,23 @@ async def scenario(): assert results[0], "both callers get the candles, not an empty chart" -def test_distinct_caller_caches_still_share_one_fetch(monkeypatch): - """Dedup keys on the upstream request, never on where the caller caches it.""" +def test_cache_bypassing_and_cached_callers_share_one_fetch(monkeypatch): + """Dedup keys on the upstream request, whether or not a caller reads the cache.""" async def scenario(): gate = asyncio.get_running_loop().create_future() gecko = _CountingGecko(gate=gate) monkeypatch.setattr(pool_data, "gecko_call", gecko) - # Three callers: two with their own private caches, one with none at all. - caches = [{}, {}, None] + # Three concurrent callers, one of them a live poll bypassing the cache: + # in flight, they still converge on one upstream request. tasks = [ asyncio.ensure_future( pool_data.fetch_ohlcv( - POOL, "meteora", timeframe="1m", user_data=cache, limit=50 + POOL, "meteora", timeframe="1m", limit=50, use_cache=use_cache ) ) - for cache in caches + for use_cache in (True, True, False) ] for _ in range(6): await asyncio.sleep(0) @@ -134,6 +134,28 @@ async def scenario(): assert all(err is None and rows == ROWS for rows, err in results) +def test_a_live_poll_skips_the_cache_read_but_refreshes_it(monkeypatch): + """use_cache=False must reach upstream even when a cached copy exists.""" + + async def scenario(): + gecko = _CountingGecko() + monkeypatch.setattr(pool_data, "gecko_call", gecko) + + await pool_data.fetch_ohlcv(POOL, "meteora", timeframe="1m", limit=50) + cached_calls = gecko.calls + await pool_data.fetch_ohlcv(POOL, "meteora", timeframe="1m", limit=50) + after_cached_read = gecko.calls + await pool_data.fetch_ohlcv( + POOL, "meteora", timeframe="1m", limit=50, use_cache=False + ) + return cached_calls, after_cached_read, gecko.calls + + first, second, third = asyncio.run(scenario()) + assert first == 1, "the first read goes upstream" + assert second == 1, "a cached reader spends nothing" + assert third == 2, "the live poll bypasses the cached copy" + + def test_a_different_window_is_not_coalesced(monkeypatch): """Same pool, different candle count: two genuinely different answers.""" @@ -165,12 +187,9 @@ async def scenario(): failing = _CountingGecko(gate=gate, fail=True) monkeypatch.setattr(pool_data, "gecko_call", failing) - cache = {} tasks = [ asyncio.ensure_future( - pool_data.fetch_ohlcv( - POOL, "meteora", timeframe="1m", user_data=cache, limit=50 - ) + pool_data.fetch_ohlcv(POOL, "meteora", timeframe="1m", limit=50) ) for _ in range(3) ] @@ -179,12 +198,10 @@ async def scenario(): gate.set_result(None) failed = await asyncio.gather(*tasks) - # Nothing was written to the caller's cache, so the retry goes upstream. + # Nothing was written to the shared cache, so the retry goes upstream. healthy = _CountingGecko() monkeypatch.setattr(pool_data, "gecko_call", healthy) - retried = await pool_data.fetch_ohlcv( - POOL, "meteora", timeframe="1m", user_data=cache, limit=50 - ) + retried = await pool_data.fetch_ohlcv(POOL, "meteora", timeframe="1m", limit=50) return failing.calls, failed, healthy.calls, retried fail_calls, failed, retry_calls, retried = asyncio.run(scenario()) diff --git a/tests/test_position_pool_attribution.py b/tests/test_position_pool_attribution.py new file mode 100644 index 000000000..756717495 --- /dev/null +++ b/tests/test_position_pool_attribution.py @@ -0,0 +1,56 @@ +"""A held position carries the executors it came from, so a pool page can place it. + +On a DEX pool page the connector *is* the network, so connector + trading pair +puts one `solana-mainnet-beta` SOL-USDC hold on every Solana SOL-USDC pool — +Orca's page and Meteora's alike — and each reads as if the position were its own. + +The API's own attribution is `executor_ids`, and an LP executor records the pool +it opened in. Passing the ids through is what lets the pool workspace tell a hold +that belongs to the pool on screen from one held in another pool of the same pair +(dropped), and from a hold no executor ties to any pool at all — a swap, whose +route the aggregator picked, which stays on the chart labelled "any pool". +""" + +from condor.web.routes.positions import _normalize_position + + +def test_the_executors_behind_a_position_survive_normalization(): + row = _normalize_position( + { + "trading_pair": "SOL-USDC", + "connector_name": "solana-mainnet-beta", + "position_side": "LONG", + "net_amount_base": 0.115990048, + "buy_breakeven_price": 86.4017516622, + "executor_count": 3, + "executor_ids": ["exec-a", "exec-b", "exec-c"], + }, + "executor", + "master_account", + ) + + assert row["executor_ids"] == ["exec-a", "exec-b", "exec-c"] + assert row["executor_count"] == 3 + + +def test_a_position_without_executors_normalizes_to_an_empty_list(): + # Bot positions come from a controller's `positions_summary`, which carries + # no ids. An absent list must not become `None`: the browser reads it as + # "nothing ties this to a pool", not as a missing field to guess about. + row = _normalize_position( + {"trading_pair": "BTC-USDT", "connector_name": "binance"}, + "bot", + "bot/main", + ) + + assert row["executor_ids"] == [] + + +def test_an_upstream_that_answers_something_else_is_not_trusted(): + row = _normalize_position( + {"trading_pair": "BTC-USDT", "executor_ids": "exec-a"}, + "executor", + "master_account", + ) + + assert row["executor_ids"] == [] diff --git a/tests/test_reports_ownership.py b/tests/test_reports_ownership.py new file mode 100644 index 000000000..bb0fba0e0 --- /dev/null +++ b/tests/test_reports_ownership.py @@ -0,0 +1,223 @@ +"""SEC-196: reports are owned, and the API authorizes reads/deletes by owner. + +The reports index was the one server-data surface in condor/web not scoped per +user: any approved caller could list every report, read every HTML body (which +carries portfolio value, PnL and positions) and delete any of them. Now every +runner stamps the authenticated principal on the entry at save time, the list +endpoints filter by it, and the id-addressed endpoints follow the sessions +idiom: own passes, admins see everything, and legacy entries with no owner are +admin-only (fail closed), never world-readable. +""" + +import asyncio +import json + +import pytest +from starlette.testclient import TestClient + +import condor.reports as rep +import condor.web.routes.reports as routes +from condor.web.app import create_app +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +USER = WebUser(id=111, username="u", first_name="U", role="user") +OTHER = WebUser(id=222, username="o", first_name="O", role="user") +ADMIN = WebUser(id=999, username="a", first_name="A", role="admin") + + +class FakeConfigManager: + def is_admin(self, user_id): + return user_id == ADMIN.id + + +def _entry(report_id: str, filename: str, owner=None, source="r"): + entry = { + "id": report_id, + "title": f"Report {report_id}", + "filename": filename, + "created_at": "2026-08-19T00:00:00+00:00", + "source_type": "routine", + "source_name": source, + "tags": [], + "agent": "condor", + } + if owner is not None: + entry["user_id"] = owner + return entry + + +@pytest.fixture +def reports_dir(tmp_path, monkeypatch): + """One report per principal, plus a legacy entry with no owner at all.""" + directory = tmp_path / "reports" + directory.mkdir() + for name, body in ( + ("mine.html", "

mine

"), + ("theirs.html", "

theirs

"), + ("legacy.html", "

legacy

"), + ): + (directory / name).write_text(body, encoding="utf-8") + index = directory / "reports_index.json" + index.write_text( + json.dumps( + [ + _entry("mine01", "mine.html", owner=USER.id, source="mine"), + _entry("their1", "theirs.html", owner=OTHER.id, source="theirs"), + # Written before SEC-196: no user_id key at all. + _entry("legacy", "legacy.html", source="legacy"), + ] + ), + encoding="utf-8", + ) + monkeypatch.setattr(rep, "CHARTS_DIR", directory) + monkeypatch.setattr(rep, "INDEX_FILE", index) + monkeypatch.setattr(routes, "get_config_manager", lambda: FakeConfigManager()) + return directory + + +def _client(user: WebUser) -> TestClient: + app = create_app() + app.dependency_overrides[get_current_user] = lambda: user + return TestClient(app) + + +# ── Listing ── + + +def test_list_returns_only_your_own_for_a_non_admin(reports_dir): + with _client(USER) as client: + payload = client.get("/api/v1/reports").json() + assert [r["id"] for r in payload["reports"]] == ["mine01"] + assert payload["total"] == 1 + + +def test_list_returns_everything_for_an_admin(reports_dir): + with _client(ADMIN) as client: + payload = client.get("/api/v1/reports").json() + assert sorted(r["id"] for r in payload["reports"]) == [ + "legacy", + "mine01", + "their1", + ] + + +def test_grouped_listing_is_scoped_the_same_way(reports_dir): + with _client(USER) as client: + groups = client.get("/api/v1/reports/latest-by-source").json() + assert [g["source_name"] for g in groups] == ["mine"] + with _client(ADMIN) as client: + groups = client.get("/api/v1/reports/latest-by-source").json() + assert sorted(g["source_name"] for g in groups) == ["legacy", "mine", "theirs"] + + +# ── Id-addressed reads and deletes ── + + +def test_owner_reads_detail_html_and_deletes_their_own(reports_dir): + with _client(USER) as client: + assert client.get("/api/v1/reports/mine01").status_code == 200 + response = client.get("/api/v1/reports/mine01/html") + assert response.status_code == 200 + assert response.text == "

mine

" + assert client.delete("/api/v1/reports/mine01").json() == {"deleted": True} + + +def test_cross_owner_report_is_refused_everywhere(reports_dir): + """A user cannot read the metadata, body, or delete another user's report.""" + with _client(USER) as client: + assert client.get("/api/v1/reports/their1").status_code == 403 + response = client.get("/api/v1/reports/their1/html") + assert response.status_code == 403 + assert "theirs" not in response.text + assert client.delete("/api/v1/reports/their1").status_code == 403 + # And nothing was deleted. + assert rep.get_report("their1") is not None + + +def test_ownerless_legacy_entry_is_admin_only(reports_dir): + """No owner recorded -> fail closed: invisible to non-admins, not deleted.""" + with _client(USER) as client: + assert client.get("/api/v1/reports/legacy").status_code == 403 + assert client.get("/api/v1/reports/legacy/html").status_code == 403 + assert client.delete("/api/v1/reports/legacy").status_code == 403 + assert rep.get_report("legacy") is not None + with _client(ADMIN) as client: + assert client.get("/api/v1/reports/legacy").status_code == 200 + assert client.get("/api/v1/reports/legacy/html").text == "

legacy

" + + +def test_admin_reads_and_deletes_across_owners(reports_dir): + with _client(ADMIN) as client: + assert client.get("/api/v1/reports/their1/html").status_code == 200 + assert client.delete("/api/v1/reports/their1").json() == {"deleted": True} + + +def test_missing_report_is_404_not_403(reports_dir): + with _client(USER) as client: + assert client.get("/api/v1/reports/nope42").status_code == 404 + + +# ── Producer stamping ── + + +def test_save_records_the_owner_from_attribute_owner(tmp_path, monkeypatch): + monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) + monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + + async def go(): + with rep.attribute_owner(USER.id): + b = rep.ReportBuilder("Owned") + b.source("routine", "r") + b.markdown("body") + await b.save() + # Outside the block the owner resets: a bare save is ownerless. + b = rep.ReportBuilder("Bare") + b.source("routine", "r") + b.markdown("body") + await b.save() + + asyncio.run(go()) + by_title = {e["title"]: e for e in rep.list_reports()[0]} + assert by_title["Owned"]["user_id"] == USER.id + assert by_title["Bare"]["user_id"] is None + + +def test_list_reports_owner_filter_drops_foreign_and_ownerless(tmp_path, monkeypatch): + monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) + monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + + async def go(): + for owner, title in ((USER.id, "Mine"), (OTHER.id, "Theirs"), (None, "Old")): + with rep.attribute_owner(owner): + b = rep.ReportBuilder(title) + b.source("routine", "r") + b.markdown("body") + await b.save() + + asyncio.run(go()) + mine, total = rep.list_reports(owner_id=USER.id) + assert [e["title"] for e in mine] == ["Mine"] + assert total == 1 + everything, _ = rep.list_reports() + assert len(everything) == 3 + + +def test_live_report_update_preserves_the_owner(tmp_path, monkeypatch): + """An in-place update keeps the user_id stamped at first save.""" + monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) + monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + + async def go(): + live = rep.LiveReport("Live", source_name="loop") + with rep.attribute_owner(USER.id): + live.builder.markdown("tick 1") + await live.update() + live.clear() + live.builder.markdown("tick 2") + with rep.attribute_owner(USER.id): + await live.update() + return live.report_id + + report_id = asyncio.run(go()) + assert rep.get_report(report_id)["user_id"] == USER.id diff --git a/tests/test_session_model_switch.py b/tests/test_session_model_switch.py index 4401369eb..a9cc0c5fe 100644 --- a/tests/test_session_model_switch.py +++ b/tests/test_session_model_switch.py @@ -98,10 +98,10 @@ def env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _Client) + monkeypatch.setattr("condor.acp.client.ACPClient", _Client) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr( "condor.runtime.binding.agent_identity_context", lambda *a, **k: "" @@ -112,7 +112,7 @@ def env(tmp_path, monkeypatch): ) monkeypatch.setattr(session_routes, "get_config_manager", FakeConfigManager) monkeypatch.setattr("condor.web.auth.get_config_manager", FakeConfigManager) - monkeypatch.setattr("handlers.agents._shared.DEFAULT_AGENT", FALLBACK) + monkeypatch.setattr("condor.llm.options.DEFAULT_AGENT", FALLBACK) monkeypatch.setattr("condor.web.routes.chat_ws.DEFAULT_AGENT", FALLBACK) AgentStore().create( diff --git a/tests/test_session_pnl_attribution.py b/tests/test_session_pnl_attribution.py index caa5e458a..b68ee2b90 100644 --- a/tests/test_session_pnl_attribution.py +++ b/tests/test_session_pnl_attribution.py @@ -12,12 +12,12 @@ import yaml -from condor.web.routes.agents import ( - AgentPerformanceModel, - _apply_bot_mode_pnl, - _current_owner_bases, - _session_ownership, +from condor.agents.attribution import ( + apply_bot_mode_pnl, + current_owner_bases, + session_ownership, ) +from condor.web.routes.agents import AgentPerformanceModel # ── Fixtures: on-disk sessions, with and without a ledger ── @@ -152,7 +152,7 @@ def test_handover_splits_pnl_at_the_takeover_instant(tmp_path): ) s1, s2 = _session(1), _session(2) - asyncio.run(_apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) assert s1.realized_pnl == 40.0 assert s2.realized_pnl == 60.0 @@ -184,7 +184,7 @@ def test_live_unrealized_goes_to_the_last_owner_not_the_newest_session(tmp_path) ) s1, s2 = _session(1), _session(2) - asyncio.run(_apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) assert s1.unrealized_pnl == 7.0 assert s1.open_count == 1 @@ -211,7 +211,7 @@ def test_session_owning_two_bots_sums_them_without_parent_folding(tmp_path): ) s1 = _session(1) - asyncio.run(_apply_bot_mode_pnl([s1], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, client)) assert s1.realized_pnl == 35.0 @@ -231,7 +231,7 @@ def test_parent_base_does_not_swallow_its_tagged_sibling(tmp_path): ) s1 = _session(1) - asyncio.run(_apply_bot_mode_pnl([s1], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, client)) # 35, not 60: the child instance is attributed to `-btc` alone. assert s1.realized_pnl == 35.0 @@ -252,7 +252,7 @@ def test_multi_controller_bot_rolls_up_to_one_session_figure(tmp_path): ) s1 = _session(1) - asyncio.run(_apply_bot_mode_pnl([s1], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, client)) assert s1.realized_pnl == 15.0 # 3 controllers × 5 @@ -270,7 +270,7 @@ def test_instance_history_is_fetched_once_per_instance(tmp_path): history={parent: [_hist_row(T3, 1.0)], child: [_hist_row(T3, 1.0)]}, ) - asyncio.run(_apply_bot_mode_pnl([_session(1), _session(2)], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([_session(1), _session(2)], tmp_path, None, client)) # Two instances, four (base, owner) windows → still exactly two fetches. assert sorted(client.history_calls) == sorted([parent, child]) @@ -405,8 +405,8 @@ def test_ledgerless_sessions_keep_session_start_tiling(tmp_path): _write_session(tmp_path, 1, {"bot_name": "dn-mm"}) _write_session(tmp_path, 2, {"bot_name": "dn-mm"}) - owned1 = _session_ownership(tmp_path, {"bot_name": "dn-mm"}, 1) - owned2 = _session_ownership(tmp_path, {"bot_name": "dn-mm"}, 2) + owned1 = session_ownership(tmp_path, {"bot_name": "dn-mm"}, 1) + owned2 = session_ownership(tmp_path, {"bot_name": "dn-mm"}, 2) assert [o.base for o in owned1] == ["dn-mm"] assert [o.origin for o in owned1] == ["legacy"] # Windows come from session start times, and session 2 started after 1. @@ -418,7 +418,7 @@ def test_ledgerless_sessions_keep_session_start_tiling(tmp_path): history={inst: [_hist_row(T0, 0.0), _hist_row(T3, 50.0)]}, ) s1, s2 = _session(1), _session(2) - asyncio.run(_apply_bot_mode_pnl([s1, s2], tmp_path, {"bot_name": "dn-mm"}, client)) + asyncio.run(apply_bot_mode_pnl([s1, s2], tmp_path, {"bot_name": "dn-mm"}, client)) # Both sessions started after T3 (they were written now), so the whole history # predates session 1's window and neither is credited — exactly what the # session-start tiling did before the ledger existed. What matters is the @@ -429,27 +429,27 @@ def test_ledgerless_sessions_keep_session_start_tiling(tmp_path): def test_ledger_wins_over_the_legacy_config_name(tmp_path): sd = _write_session(tmp_path, 1, {"bot_name": "dn-mm"}) _write_ledger(sd, {"ns-other": 123.0}) - owned = _session_ownership(tmp_path, {"bot_name": "dn-mm"}, 1) + owned = session_ownership(tmp_path, {"bot_name": "dn-mm"}, 1) assert [(o.base, o.since) for o in owned] == [("ns-other", 123.0)] def test_direct_executor_strategy_owns_nothing(tmp_path): _write_session(tmp_path, 1, {"bot_name": ""}) - assert _session_ownership(tmp_path, {}, 1) == [] + assert session_ownership(tmp_path, {}, 1) == [] # ── Current-owner lookup (per-session executors view) ── -def test_current_owner_bases_follows_the_ledger_not_the_session_number(tmp_path): +def testcurrent_owner_bases_follows_the_ledger_not_the_session_number(tmp_path): sd1 = _write_session(tmp_path, 1) sd2 = _write_session(tmp_path, 2) _write_ledger(sd1, {"ns-a": 100.0, "ns-b": 100.0}) _write_ledger(sd2, {"ns-a": 200.0}) # session 2 adopted only ns-a nums = [1, 2] - assert _current_owner_bases(tmp_path, None, nums, 1) == ["ns-b"] - assert _current_owner_bases(tmp_path, None, nums, 2) == ["ns-a"] + assert current_owner_bases(tmp_path, None, nums, 1) == ["ns-b"] + assert current_owner_bases(tmp_path, None, nums, 2) == ["ns-a"] # ── Whole-server snapshot is fetched once per rollup, not once per strategy ── @@ -458,11 +458,14 @@ def test_current_owner_bases_follows_the_ledger_not_the_session_number(tmp_path) def test_rollup_fans_out_one_snapshot_call_for_all_strategies(tmp_path): """N strategies on one server share a single whole-server snapshot fetch. - ``list_agents`` gathers every strategy's ``_apply_bot_mode_pnl`` at once, and + ``list_agents`` gathers every strategy's ``apply_bot_mode_pnl`` at once, and ``get_latest_controller_performance()`` returns the same whole-server payload to each of them — so the fan-out must collapse to one round-trip. """ - from condor.fetchers.bot_performance import clear_snapshot_cache + from condor.fetchers.bot_performance import ( + clear_history_cache, + clear_snapshot_cache, + ) class _CountingClient(_FakeClient): def __init__(self, *args, **kwargs): @@ -489,16 +492,18 @@ async def get_latest_controller_performance(self): async def _go(): await asyncio.gather( *[ - _apply_bot_mode_pnl(sessions, sdir, None, client) + apply_bot_mode_pnl(sessions, sdir, None, client) for sdir, sessions in strategies ] ) clear_snapshot_cache() + clear_history_cache() try: asyncio.run(_go()) finally: clear_snapshot_cache() + clear_history_cache() assert client.snapshot_calls == 1 # Every strategy still gets its attribution from that one snapshot. @@ -533,7 +538,7 @@ def test_long_lived_bot_does_not_dump_old_pnl_on_one_session(tmp_path): ) s1, s2, s3 = _session(1), _session(2), _session(3) - asyncio.run(_apply_bot_mode_pnl([s1, s2, s3], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1, s2, s3], tmp_path, None, client)) # Each session keeps only what it earned; none is starved and none inherits. assert s1.realized_pnl == 40.0 # days 0–4 @@ -569,7 +574,7 @@ def test_fees_are_sliced_per_session_not_dumped_on_the_last_owner(tmp_path): ) s1, s2 = _session(1), _session(2) - asyncio.run(_apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) assert s1.fees == 4.0 assert s2.fees == 5.0 @@ -595,7 +600,7 @@ def test_fees_fall_back_to_live_figure_when_history_has_no_fee_column(tmp_path): ) s1 = _session(1) - asyncio.run(_apply_bot_mode_pnl([s1], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, client)) assert s1.fees == 3.5 @@ -623,7 +628,7 @@ def test_released_session_stops_accruing_a_surviving_bot(tmp_path): ) s1 = _session(1) - asyncio.run(_apply_bot_mode_pnl([s1], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, client)) assert s1.realized_pnl == 40.0 # only up to the release assert s1.unrealized_pnl == 0.0 # an ended session holds no open book @@ -631,11 +636,11 @@ def test_released_session_stops_accruing_a_surviving_bot(tmp_path): def test_released_session_is_not_the_current_owner(tmp_path): - """_current_owner_bases and the slicing agree that a released bot is unowned.""" + """current_owner_bases and the slicing agree that a released bot is unowned.""" sd1 = _write_session(tmp_path, 1) _write_ledger(sd1, {"ns-bot": _epoch(T0)}, until={"ns-bot": _epoch(T2)}) - assert _current_owner_bases(tmp_path, None, [1], 1) == [] + assert current_owner_bases(tmp_path, None, [1], 1) == [] def test_release_is_idempotent_and_reopens_on_further_use(tmp_path): @@ -683,3 +688,65 @@ def test_release_preserves_namespace_for_a_ledger_opened_without_one(tmp_path): assert reopened.namespace == "ns" # not blanked by the release write assert reopened.declared == ["legacy-bot"] assert read_owned(sd)[0].until == 200.0 + + +# ── One engine, two surfaces (ARCH-191) ── + + +def test_rollup_and_agent_view_agree_on_an_adopted_bot(): + """The dashboard rollup and the agent's own fetch share one attribution engine. + + A session that adopts a long-running bot must be credited the same + realized / volume / trades / fees on both surfaces: the web strategy rollup + (``apply_bot_mode_pnl``) and the agent's own ``fetch_agent_performance``. + Both now fold through ``condor.agents.attribution``, so agreement is + structural — this test pins the contract over a synthetic history. + """ + import tempfile + from types import SimpleNamespace + + from condor.agents.performance import fetch_agent_performance + + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + sd1 = _write_session(tmp_path, 1) + # The bot ran since T0; this session only took it over at T2. + _write_ledger(sd1, {"ns-bot": _epoch(T2)}) + + inst = "ns-bot-20260701-000000" + closes = {"CloseType.TAKE_PROFIT": 3} + history = { + inst: [ + _hist_row(T0, 0.0, cum_volume=0.0, cum_fees=0.0), + _hist_row(T2, 40.0, cum_volume=4000.0, cum_fees=4.0), + _hist_row(T3, 100.0, cum_volume=9000.0, cum_fees=9.0, closes=closes), + ] + } + snapshots = [_snap(inst, T3, realized=100.0, unrealized=7.0)] + + # Web rollup surface. + rollup_client = _FakeClient(snapshots=snapshots, history=history) + s1 = _session(1) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, rollup_client)) + + # Agent's own surface, over the same ownership window. + async def _no_executors(**_kw): + return [] + + agent_client = _FakeClient(snapshots=snapshots, history=history) + agent_client.executors = SimpleNamespace(search_executors=_no_executors) + owned = session_ownership(tmp_path, None, 1) + since = min(b.since for b in owned) + detail = asyncio.run( + fetch_agent_performance( + agent_client, "a_1", bot_names=[b.base for b in owned], since=since + ) + ) + + # Post-takeover only — the inherited 40/4000/4.0 belongs to nobody here. + assert s1.realized_pnl == detail.realized_pnl == 60.0 + assert s1.volume == detail.volume == 5000.0 + assert s1.fees == detail.fees == 5.0 + assert s1.trade_count == detail.trade_count == 3 + assert s1.unrealized_pnl == detail.unrealized_pnl == 7.0 + assert s1.fees_known and detail.fees_known diff --git a/tests/test_session_server_access.py b/tests/test_session_server_access.py index a8ab579fe..49392307c 100644 --- a/tests/test_session_server_access.py +++ b/tests/test_session_server_access.py @@ -220,11 +220,11 @@ async def prompt(self, text): @pytest.fixture def registry(monkeypatch): monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _FakeClient) - monkeypatch.setattr(session_module, "PydanticAIClient", _FakeClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _FakeClient) + monkeypatch.setattr("condor.acp.pydantic_ai_client.PydanticAIClient", _FakeClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) return session_module diff --git a/tests/test_session_server_switch.py b/tests/test_session_server_switch.py index 29b463c24..b1bcee360 100644 --- a/tests/test_session_server_switch.py +++ b/tests/test_session_server_switch.py @@ -83,14 +83,14 @@ def env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _Client) + monkeypatch.setattr("condor.acp.client.ACPClient", _Client) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr(routes, "get_config_manager", lambda: FakeConfigManager()) monkeypatch.setattr( "condor.web.auth.get_config_manager", lambda: FakeConfigManager() ) monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) diff --git a/tests/test_session_switch.py b/tests/test_session_switch.py index afad27bfc..61edb3251 100644 --- a/tests/test_session_switch.py +++ b/tests/test_session_switch.py @@ -68,11 +68,11 @@ def env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _Client) + monkeypatch.setattr("condor.acp.client.ACPClient", _Client) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr(routes, "get_config_manager", lambda: FakeConfigManager()) monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) diff --git a/tests/test_shutdown.py b/tests/test_shutdown.py index 273c3bd57..40ae2fe28 100644 --- a/tests/test_shutdown.py +++ b/tests/test_shutdown.py @@ -307,8 +307,12 @@ def spy(path, text, **kwargs): return recorded -def _with_real_journal(engine, tmp_path) -> JournalManager: +def _with_real_journal(engine, tmp_path, journal_writes=None) -> JournalManager: engine.journal = JournalManager(engine.agent_id, session_dir=tmp_path) + # Creating the journal writes the template (atomically, CORR-193); that + # bootstrap write is not part of the winddown behavior under test. + if journal_writes is not None: + journal_writes.clear() return engine.journal @@ -318,7 +322,7 @@ def test_winddown_rewrites_the_journal_once_at_the_end( """The terminal pair (append_action + record_tick) wrote the file twice.""" running = [{"id": "e_perp", "connector": "binance_perpetual"}] engine, _client, _notes = _fake_engine(running, [[]], monkeypatch, tmp_path) - _with_real_journal(engine, tmp_path) + _with_real_journal(engine, tmp_path, journal_writes) asyncio.run(run_shutdown(engine, "breach")) @@ -333,7 +337,7 @@ def test_winddown_without_client_rewrites_the_journal_once( tmp_path, monkeypatch, journal_writes ): engine, _client, _notes = _fake_engine([], [[]], monkeypatch, tmp_path) - _with_real_journal(engine, tmp_path) + _with_real_journal(engine, tmp_path, journal_writes) async def _no_client(): return None diff --git a/tests/test_skill_store.py b/tests/test_skill_store.py index 8d6cf27f4..5bcaaa4b0 100644 --- a/tests/test_skill_store.py +++ b/tests/test_skill_store.py @@ -586,7 +586,7 @@ def test_concurrent_writers_never_leave_a_torn_file(project_root): # always parse cleanly, never a torn/interleaved one. import threading - from condor.memory.store import _parse_frontmatter + from condor.frontmatter import parse_frontmatter s = SkillStore() target = s.skills_dir / "shared" / "SKILL.md" @@ -608,7 +608,7 @@ def writer(text): for t in threads: t.join() - meta, body = _parse_frontmatter(target.read_text()) + meta, body = parse_frontmatter(target.read_text()) assert meta.get("name") == "shared" assert body == "x" * 5000 assert list(target.parent.glob("*.tmp")) == [] diff --git a/tests/test_strategy_perf_cache.py b/tests/test_strategy_perf_cache.py index e2feb8313..c5869ae4f 100644 --- a/tests/test_strategy_perf_cache.py +++ b/tests/test_strategy_perf_cache.py @@ -7,7 +7,7 @@ """ import asyncio -from collections import Counter +from collections import Counter, OrderedDict from types import SimpleNamespace import pytest @@ -65,7 +65,7 @@ def _make_sessions(strategy_dir, nums): def perf_env(tmp_path, monkeypatch): """Isolated caches + engine registry + a client factory hook.""" monkeypatch.setattr(agents_routes, "_PERF_CACHE", {}) - monkeypatch.setattr(agents_routes, "_CLOSED_PERF_CACHE", {}) + monkeypatch.setattr(agents_routes, "_CLOSED_PERF_CACHE", OrderedDict()) # The running-engine registry moved into the supervisor (FEAT-012). monkeypatch.setattr(loops_module.get_supervisor(), "_engines", {}) @@ -205,3 +205,61 @@ def test_reactivated_id_evicts_frozen_entry(perf_env): _compute(strategy_dir) assert api.calls[f"{RUN_KEY}_1"] == 2 assert f"{RUN_KEY}_1" not in agents_routes._CLOSED_PERF_CACHE + + +def test_closed_cache_never_exceeds_cap(perf_env, monkeypatch): + """PERF-186: the frozen cache is a bounded LRU, not a monotonic dict.""" + cap = 4 + monkeypatch.setattr(agents_routes, "_CLOSED_PERF_CACHE_MAX", cap) + n_sessions = cap + 4 # 7 closed + the newest (never frozen) + strategy_dir, use_client = perf_env + _make_sessions(strategy_dir, range(1, n_sessions + 1)) + api = _FakeExecutorsApi( + {f"{RUN_KEY}_{n}": [_closed_executor(1.0)] for n in range(1, n_sessions + 1)} + ) + use_client(_FakeClient(api)) + + sessions, totals = _compute(strategy_dir) + + assert len(sessions) == n_sessions # all sessions still render + # Exactly `cap` closed sessions survive (freeze order follows iterdir(), + # so which ones is filesystem-dependent); the newest is never frozen. + assert len(agents_routes._CLOSED_PERF_CACHE) == cap + closed_ids = {f"{RUN_KEY}_{n}" for n in range(1, n_sessions)} + assert set(agents_routes._CLOSED_PERF_CACHE) <= closed_ids + assert totals["total_pnl"] == pytest.approx(float(n_sessions)) + + +def test_evicted_session_refetched_and_refrozen(perf_env, monkeypatch): + """An LRU-evicted closed session flows through the fetch path again.""" + cap = 2 + monkeypatch.setattr(agents_routes, "_CLOSED_PERF_CACHE_MAX", cap) + strategy_dir, use_client = perf_env + _make_sessions(strategy_dir, [1, 2, 3, 4]) + api = _FakeExecutorsApi( + {f"{RUN_KEY}_{n}": [_closed_executor(float(n))] for n in [1, 2, 3, 4]} + ) + use_client(_FakeClient(api)) + + sessions1, totals1 = _compute(strategy_dir) + # Cap 2 with 3 closed sessions: exactly one closed session got evicted + # (freeze order follows iterdir(), so which one is filesystem-dependent). + closed_ids = {f"{RUN_KEY}_{n}" for n in [1, 2, 3]} + cached = set(agents_routes._CLOSED_PERF_CACHE) + evicted = closed_ids - cached + assert len(cached) == cap + assert len(evicted) == 1 + + agents_routes._PERF_CACHE.clear() + sessions2, totals2 = _compute(strategy_dir) + + # The evicted session was re-fetched; the still-frozen ones were not. + for aid in evicted: + assert api.calls[aid] == 2 + for aid in cached: + assert api.calls[aid] == 1 + assert sorted(s.agent_id for s in sessions2) == sorted( + s.agent_id for s in sessions1 + ) + assert totals2 == totals1 + assert totals2["total_pnl"] == pytest.approx(10.0) diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 739844a1e..6294af3c8 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -25,7 +25,6 @@ def install(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) monkeypatch.setattr("utils.config.CONDOR_TELEMETRY", None, raising=False) - monkeypatch.setattr("utils.config.CONDOR_TELEMETRY_URL", None, raising=False) monkeypatch.setattr( "condor.agents.agent._DATA_ROOT", str(tmp_path / "data"), raising=False ) @@ -46,23 +45,24 @@ def _grant(level="usage"): consent.grant(level) -# ── The default is silence ─────────────────────────────────────────────── +# ── The default is the ping floor ──────────────────────────────────────── -def test_a_fresh_install_is_off_and_stays_off(install): - """No consent recorded means no telemetry — not "buffered until asked".""" +def test_a_fresh_install_pings_and_says_nothing_more(install): + """No consent recorded means install counting only — the ping floor.""" assert consent.state() == consent.UNKNOWN - assert consent.level() == consent.OFF + assert consent.level() == consent.PING for name in schema.EVENTS: emitter.emit(name, surface="telegram") - assert emitter.buffered() == 0 - assert emitter.dropped() == 0 + names = {e["name"] for e in emitter.drain()[0]} + assert names == set(schema.PING_EVENTS) -def test_nothing_is_written_to_disk_when_off(install): - """Acceptance criterion: no condor/.runtime/telemetry directory is created.""" +def test_usage_events_write_nothing_to_disk_at_the_ping_floor(install): + """Acceptance criterion: no condor/.runtime/telemetry directory is created + by activity that the install has no consent to report.""" emitter.set_hosted(False) # the spooling path, which is the one that does I/O emitter.emit("command", name="portfolio", surface="telegram") assert not outbox.root().exists() @@ -84,7 +84,7 @@ def test_env_off_overrides_a_granted_consent(install, monkeypatch): def test_a_nonsense_env_level_is_ignored_not_obeyed(install, monkeypatch): monkeypatch.setattr("utils.config.CONDOR_TELEMETRY", "yes-please", raising=False) consent.refresh() - assert consent.level() == consent.OFF + assert consent.level() == consent.PING # the floor, not whatever was typed def test_ping_level_sends_only_the_adoption_events(install): @@ -100,8 +100,8 @@ def test_ping_level_sends_only_the_adoption_events(install): # ── Nothing transmits ──────────────────────────────────────────────────── -def test_flush_is_a_no_op_without_consent(install, monkeypatch): - """A full session of activity must produce zero outbound requests.""" +def test_usage_activity_never_transmits_without_consent(install, monkeypatch): + """A full session of activity must produce zero outbound usage events.""" calls = [] monkeypatch.setattr(outbox, "post", lambda env: calls.append(env)) @@ -114,12 +114,39 @@ def test_flush_is_a_no_op_without_consent(install, monkeypatch): assert not outbox.root().exists() -def test_no_endpoint_means_events_only_ever_reach_a_local_file(install): - """The shipped state: consent granted, but nowhere to send. Nothing leaves.""" +def test_the_ping_floor_actually_flushes_without_an_answer(install, monkeypatch): + """An unanswered install still delivers its adoption events — the count is + only worth anything if it leaves the machine.""" + sent = [] + + async def ok_post(envelope): + sent.append(envelope) + return True + + monkeypatch.setattr(outbox, "endpoint", lambda: "http://collector.invalid") + monkeypatch.setattr(outbox, "post", ok_post) + + assert consent.state() == consent.UNKNOWN + emitter.emit("install") + assert asyncio.run(emitter.flush("test")) == 1 + assert [e["name"] for env in sent for e in env["events"]] == ["install"] + assert sent[0]["level"] == consent.PING + + +def test_the_endpoint_is_fixed_and_not_configurable(install, monkeypatch): + """The collector address is compiled in; no env var may redirect it.""" + monkeypatch.setenv("CONDOR_TELEMETRY_URL", "http://attacker.invalid/v1/events") + + assert outbox.endpoint() == outbox.COLLECTOR_URL + assert outbox.COLLECTOR_URL == "https://telemetry.hummingbot.org/v1/events" + + +def test_no_endpoint_means_events_only_ever_reach_a_local_file(install, monkeypatch): + """With delivery impossible, events park in the capped outbox instead of leaving.""" _grant("usage") + monkeypatch.setattr(outbox, "endpoint", lambda: None) emitter.emit("command", name="portfolio", surface="telegram") - assert outbox.endpoint() is None assert asyncio.run(emitter.flush("test")) == 0 stashed = outbox.take_stashed() @@ -164,7 +191,7 @@ def test_the_outbox_respects_its_cap(install): assert len(outbox.take_stashed()) == outbox.MAX_OUTBOX_EVENTS -def test_denying_consent_destroys_what_was_collected(install): +def test_withdrawing_usage_consent_destroys_what_was_collected(install): _grant("usage") emitter.emit("command", name="portfolio", surface="telegram") outbox.stash( @@ -178,13 +205,25 @@ def test_denying_consent_destroys_what_was_collected(install): ) assert outbox.outbox_path().exists() - consent.deny() + consent.set_level("ping") - assert consent.level() == consent.OFF + assert consent.level() == consent.PING assert emitter.buffered() == 0 assert not outbox.outbox_path().exists() +def test_off_is_not_a_grantable_answer(install): + """The consent form has no "off": a stale or unrecognized answer lands on + ping, the floor, and set_level rejects "off" outright.""" + from config_manager import get_config_manager + + get_config_manager() # materialize config.yml in the tmp cwd + assert consent.grant("off") == consent.PING + assert consent.state() == consent.GRANTED + assert consent.set_level("off") == consent.PING # rejected, level unchanged + assert consent.level() == consent.PING + + # ── The leak test ──────────────────────────────────────────────────────── FORBIDDEN = { @@ -386,6 +425,89 @@ def test_the_install_context_carries_no_identity(install): assert identifying not in blob +def test_the_envelope_says_which_surface_the_install_runs(install, monkeypatch): + """`mode` is one of two fixed names — the adoption signal local mode was + built for, and nothing that could point at a host or a chat.""" + monkeypatch.setattr("utils.config.CONDOR_MODE", "local", raising=False) + assert context.config_shape()["mode"] == "local" + monkeypatch.setattr("utils.config.CONDOR_MODE", "telegram", raising=False) + assert context.config_shape()["mode"] == "telegram" + + +# ── Every install is counted ───────────────────────────────────────────── + + +def test_a_fresh_install_is_counted_without_anyone_answering(install): + """The claim that gives this feature its floor. `install` used to be emitted + only from the Telegram consent callback, so an install that never answered + — every local-mode one — was never counted at all.""" + import condor.telemetry as telemetry + + telemetry.init(hosted=True) + + assert consent.state() == consent.UNKNOWN # nobody answered anything + assert [e["name"] for e in emitter.drain()[0]] == ["install"] + + +def test_an_install_is_counted_once_however_many_times_it_boots(install): + import condor.telemetry as telemetry + + telemetry.init(hosted=True) + emitter.drain() + + telemetry.init(hosted=True) + telemetry.init(hosted=True) + assert emitter.drain()[0] == [] + + +def test_answering_the_prompt_does_not_count_the_install_a_second_time(install): + import condor.telemetry as telemetry + + telemetry.init(hosted=True) + emitter.drain() + + consent.grant("usage") + assert consent.mark_install_reported() is False + + +def test_an_install_silenced_by_the_environment_is_never_counted(install, monkeypatch): + """`CONDOR_TELEMETRY=off` is the kill switch, and counting is not exempt + from it — nothing is recorded and config.yml is left as it was found.""" + monkeypatch.setattr("utils.config.CONDOR_TELEMETRY", "off", raising=False) + import condor.telemetry as telemetry + + assert telemetry.init(hosted=True) == consent.OFF + assert emitter.drain()[0] == [] + assert not (install / "config.yml").exists() + + +# ── Asking, on both surfaces ───────────────────────────────────────────── + + +def test_telegram_and_the_dashboard_disclose_exactly_the_same_thing(): + """Two prompts mean two chances to disagree about what is collected. The + dashboard card renders `DISCLOSURE`; the Telegram message is built from it.""" + from condor.telemetry.prompt import _TEXT, DISCLOSURE + + assert DISCLOSURE["headline"] in _TEXT + assert DISCLOSURE["always_on"] in _TEXT + assert DISCLOSURE["optional"] in _TEXT + assert DISCLOSURE["doc"] in _TEXT + for never in DISCLOSURE["never"]: + assert never in _TEXT + + +def test_the_prompt_offers_no_way_to_switch_counting_off(): + """`off` is the operator's env kill switch, never a button — on either + surface. A stray option here would quietly make the floor optional.""" + from condor.telemetry.prompt import DISCLOSURE, OPTIONS + + levels = {option["level"] for option in OPTIONS} + assert levels == set(consent.ANSWER_LEVELS) + assert consent.OFF not in levels + assert [o["level"] for o in DISCLOSURE["options"]] == [o["level"] for o in OPTIONS] + + # ── Out of process ─────────────────────────────────────────────────────── @@ -428,8 +550,9 @@ def test_emit_costs_under_50_microseconds_on_the_hot_path(install): assert per_call < 50e-6, f"emit() took {per_call * 1e6:.1f}us" -def test_emit_is_free_when_off(install): - """The gate a trading path actually pays for.""" +def test_emit_is_free_below_usage(install): + """The gate a trading path actually pays for: at the ping floor a usage + event is rejected by the schema gate before any work happens.""" iterations = 2000 started = time.perf_counter() for _ in range(iterations): @@ -491,7 +614,8 @@ def test_a_callback_reports_module_and_verb_only(install): def test_the_telegram_tap_emits_nothing_before_consent(install): - """The acceptance criterion, at the seam rather than at the emitter.""" + """The acceptance criterion, at the seam rather than at the emitter: + commands and callbacks are usage events, and the ping floor drops them.""" assert consent.state() == consent.UNKNOWN for update in ( _FakeUpdate(text="/portfolio"), diff --git a/tests/test_telemetry_consent_web.py b/tests/test_telemetry_consent_web.py new file mode 100644 index 000000000..b57880216 --- /dev/null +++ b/tests/test_telemetry_consent_web.py @@ -0,0 +1,136 @@ +"""The dashboard half of the telemetry consent prompt (FEAT-023). + +A Telegram install is asked once, next to the boot notification. An install +running without Telegram has no bot to be asked through, so before this the +question could not be put to it at all: consent stayed ``unknown`` forever and +no amount of willingness could move it off the ping floor. These tests pin the +contract the dashboard consent card is built on. +""" + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.settings as settings_routes +from condor.telemetry import consent +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +ADMIN = WebUser(id=1, username="admin", first_name="A", role="admin") +TRADER = WebUser(id=2, username="trader", first_name="T", role="user") + + +class FakeConfigManager: + def is_admin(self, user_id): + return user_id == ADMIN.id + + +@pytest.fixture +def app(tmp_path, monkeypatch): + """An isolated install, so the answers these tests give are written to a + config.yml that dies with the test.""" + import config_manager as cm_module + from condor.telemetry import emitter + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("utils.config.CONDOR_TELEMETRY", None, raising=False) + cm_module.ConfigManager.reset_instance() + emitter.discard_buffer() + consent.refresh() + + # Only the route's own authorization check is faked; `consent` keeps its + # real ConfigManager so a PUT genuinely lands on disk. + monkeypatch.setattr(settings_routes, "get_config_manager", FakeConfigManager) + + api = FastAPI() + api.include_router(settings_routes.router) + yield api + + cm_module.ConfigManager.reset_instance() + emitter.discard_buffer() + consent.refresh() + + +def as_user(app, user): + app.dependency_overrides[get_current_user] = lambda: user + return TestClient(app) + + +def test_an_install_that_never_answered_reports_itself_as_unanswered(app): + """`unknown` plus `can_change` is the whole trigger for the consent card.""" + body = as_user(app, ADMIN).get("/settings/telemetry").json() + + assert body["consent"] == "unknown" + assert body["level"] == "ping" # counted meanwhile — the floor + assert body["can_change"] is True + assert body["env_overridden"] is False + + +def test_every_seat_may_read_what_the_install_shares(app): + """Transparency is not an admin feature. Answering is.""" + body = as_user(app, TRADER).get("/settings/telemetry").json() + + assert body["consent"] == "unknown" + assert body["can_change"] is False + assert body["disclosure"]["never"] # the same list, for everyone + + +def test_the_card_is_served_the_same_words_telegram_sends(app): + from condor.telemetry.prompt import DISCLOSURE + + body = as_user(app, ADMIN).get("/settings/telemetry").json() + assert body["disclosure"] == DISCLOSURE + + +def test_a_non_admin_cannot_answer_for_the_install(app): + resp = as_user(app, TRADER).put("/settings/telemetry", params={"level": "usage"}) + + assert resp.status_code == 403 + assert consent.state() == consent.UNKNOWN + + +def test_answering_in_the_dashboard_retires_the_question(app): + """The card's whole job: move consent off `unknown` and record the level. + Once it is granted the banner stops rendering, on this surface and the + other one.""" + resp = as_user(app, ADMIN).put("/settings/telemetry", params={"level": "usage"}) + + assert resp.status_code == 200 + assert resp.json() == {"level": "usage", "consent": "granted"} + assert consent.state() == consent.GRANTED + assert consent.level() == consent.USAGE + + after = as_user(app, ADMIN).get("/settings/telemetry").json() + assert after["consent"] == "granted" + + +def test_only_counting_is_a_real_answer_too(app): + """ "Only count my install" is an answer, not a refusal to answer — it has + to retire the prompt, or the admin gets asked again forever.""" + as_user(app, ADMIN).put("/settings/telemetry", params={"level": "ping"}) + + assert consent.state() == consent.GRANTED + assert consent.level() == consent.PING + + +def test_the_dashboard_cannot_switch_install_counting_off(app): + resp = as_user(app, ADMIN).put("/settings/telemetry", params={"level": "off"}) + + assert resp.status_code == 400 + assert consent.state() == consent.UNKNOWN + + +def test_an_environment_pinned_install_is_not_asked_and_cannot_be_answered( + app, monkeypatch +): + """With `CONDOR_TELEMETRY` set there is no question left to put: the card + reports the override instead of offering buttons, and a PUT is refused.""" + monkeypatch.setattr("utils.config.CONDOR_TELEMETRY", "off", raising=False) + consent.refresh() + + body = as_user(app, ADMIN).get("/settings/telemetry").json() + assert body["env_overridden"] is True + assert body["level"] == "off" + + resp = as_user(app, ADMIN).put("/settings/telemetry", params={"level": "usage"}) + assert resp.status_code == 409 diff --git a/tests/test_web_report_serving.py b/tests/test_web_report_serving.py index cd7171e0e..70cd47787 100644 --- a/tests/test_web_report_serving.py +++ b/tests/test_web_report_serving.py @@ -12,12 +12,19 @@ from starlette.testclient import TestClient import condor.reports as reports +import condor.web.routes.reports as reports_routes from condor.web.app import create_app from condor.web.auth import get_current_user from condor.web.models import WebUser USER = WebUser(id=111, username="u", first_name="U", role="user") + +class _NoAdmins: + def is_admin(self, user_id): + return False + + BODY = "

Report

" # Distinctive markers so a leak is unmistakable and cannot be confused with a # 404 detail message that happens to echo the requested path back. @@ -38,6 +45,10 @@ def reports_dir(tmp_path, monkeypatch): index.write_text( json.dumps( [ + # Entries are stamped with USER's id: since SEC-196 an entry + # with no owner is admin-only, and these tests exercise the + # serving/path-containment logic, not the ownership gate + # (which has its own tests in test_reports_ownership.py). { "id": "abc123", "title": "Sample", @@ -46,6 +57,7 @@ def reports_dir(tmp_path, monkeypatch): "source_type": "routine", "source_name": "sample", "tags": [], + "user_id": 111, }, # Hostile index entries: the filename must never escape the # reports directory, and must be .html, even if the index lies. @@ -57,6 +69,7 @@ def reports_dir(tmp_path, monkeypatch): "source_type": "routine", "source_name": "escape", "tags": [], + "user_id": 111, }, { "id": "nothtml", @@ -66,6 +79,7 @@ def reports_dir(tmp_path, monkeypatch): "source_type": "routine", "source_name": "nothtml", "tags": [], + "user_id": 111, }, ] ), @@ -73,6 +87,9 @@ def reports_dir(tmp_path, monkeypatch): ) monkeypatch.setattr(reports, "CHARTS_DIR", directory) monkeypatch.setattr(reports, "INDEX_FILE", index) + # Ownership (SEC-196) is orthogonal here: USER owns every seeded entry and + # is not an admin, so these tests keep exercising serving/containment. + monkeypatch.setattr(reports_routes, "get_config_manager", lambda: _NoAdmins()) return directory diff --git a/tests/test_web_servers_parallel_status.py b/tests/test_web_servers_parallel_status.py index 2dd187601..fd4f3de1e 100644 --- a/tests/test_web_servers_parallel_status.py +++ b/tests/test_web_servers_parallel_status.py @@ -29,9 +29,10 @@ def __init__(self, value): class FakeConfigManager: - def __init__(self, servers, permission="owner"): + def __init__(self, servers, permission="owner", default=None): self._servers = servers self._permission = permission + self._default = default def list_accessible_servers(self, user_id): return self._servers @@ -39,6 +40,9 @@ def list_accessible_servers(self, user_id): def get_server_permission(self, user_id, name): return FakePermission(self._permission) if self._permission else None + def get_chat_default_server(self, chat_id): + return self._default + class FakeSDS: """Cache always misses; each fetch sleeps like a connect timeout would.""" @@ -119,6 +123,7 @@ def test_all_online_response_is_unchanged(monkeypatch): "port": 8000, "online": True, "permission": "owner", + "is_default": False, }, { "name": "bravo", @@ -126,6 +131,7 @@ def test_all_online_response_is_unchanged(monkeypatch): "port": 8001, "online": True, "permission": "owner", + "is_default": False, }, { "name": "charlie", @@ -133,6 +139,7 @@ def test_all_online_response_is_unchanged(monkeypatch): "port": 8002, "online": True, "permission": "owner", + "is_default": False, }, ] @@ -195,3 +202,28 @@ def test_missing_permission_falls_back_to_trader(monkeypatch): client = build_client(monkeypatch, cm, sds) assert {r["permission"] for r in client.get("/servers").json()} == {"trader"} + + +def test_the_chat_default_server_is_the_only_one_flagged(monkeypatch): + """The selector seeds itself from this flag, so exactly one row carries it. + + Without it the dashboard could not tell which server Telegram's + ``chat_defaults`` points at, and the Settings star had nothing to fill in — + the click looked like it did nothing at all. + """ + sds = FakeSDS(statuses={n: {"status": "online"} for n in three_servers()}, delay=0) + cm = FakeConfigManager(three_servers(), default="bravo") + client = build_client(monkeypatch, cm, sds) + + assert {r["name"]: r["is_default"] for r in client.get("/servers").json()} == { + "alpha": False, + "bravo": True, + "charlie": False, + } + + +def test_no_default_flags_nothing(monkeypatch): + sds = FakeSDS(statuses={n: {"status": "online"} for n in three_servers()}, delay=0) + client = build_client(monkeypatch, FakeConfigManager(three_servers()), sds) + + assert not any(r["is_default"] for r in client.get("/servers").json()) diff --git a/utils/auth.py b/utils/auth.py index cdb7ddb37..bcc47e966 100644 --- a/utils/auth.py +++ b/utils/auth.py @@ -36,6 +36,35 @@ async def _notify_admin_new_user( logger.warning(f"Failed to notify admin about new user: {e}") +def _refresh_active_server(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + """Re-sync the cached ``active_server`` from config.yml before the handler runs. + + ``chat_defaults`` in config.yml is where every surface records the chosen + default, but most handlers read the ``user_data`` copy of it + (``get_active_server``) rather than resolving through + ``get_effective_server``. That copy is only rewritten when something happens + to resolve — so a default picked on the web dashboard stayed invisible to + /portfolio, /bots and the gateway screens until the user opened /servers and + incidentally triggered a resolve. + + Running the resolve here, at the one gate every Telegram update passes + through, keeps the copy true for whatever the handler does with it. It is a + dict lookup on the already-loaded config in the common case; the write only + happens on an actual change. + """ + if context.user_data is None: + return + chat = getattr(update, "effective_chat", None) + if chat is None: + return + try: + from config_manager import get_effective_server + + get_effective_server(chat.id, context.user_data) + except Exception as e: # never block a command over a preference refresh + logger.debug("Could not refresh active server: %s", e) + + def restricted(func): """ Decorator that checks if user is approved. @@ -98,6 +127,7 @@ async def wrapped( # User is approved (USER or ADMIN role) # Store user_id in context for access control in subsequent calls context.user_data["_user_id"] = user_id + _refresh_active_server(update, context) return await func(update, context, *args, **kwargs) return wrapped diff --git a/utils/config.py b/utils/config.py index c0d6e160b..ca4d85919 100644 --- a/utils/config.py +++ b/utils/config.py @@ -1,19 +1,39 @@ import os +from typing import Optional from urllib.parse import urlparse from dotenv import load_dotenv load_dotenv() -# Primary admin user ID - this user has full control over the bot -# Set via ADMIN_USER_ID environment variable -ADMIN_USER_ID = None -_admin_id_str = os.environ.get("ADMIN_USER_ID", "").strip() -if _admin_id_str: + +class ConfigError(RuntimeError): + """The process is configured in a way it must not start with.""" + + +def resolve_admin_id(env=None) -> Optional[int]: + """The primary admin's user id, or ``None`` when it is unset or unusable. + + Junk resolves to ``None`` rather than raising here, because this runs at + import time and every module reads this file; the refusal happens once, at + boot, in :func:`check_startup_config`. Non-positive ids count as unusable: + several call sites read a falsy user or chat id as "absent", so ``0`` would + silently address nobody. + """ + env = os.environ if env is None else env + raw = (env.get("ADMIN_USER_ID") or "").strip() + if not raw: + return None try: - ADMIN_USER_ID = int(_admin_id_str) + value = int(raw) except ValueError: - pass + return None + return value if value > 0 else None + + +# Primary admin user ID - this user has full control over the bot +# Set via ADMIN_USER_ID environment variable +ADMIN_USER_ID = resolve_admin_id() TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN") @@ -42,7 +62,146 @@ # usage - the full allowlisted taxonomy in condor/telemetry/schema.py. CONDOR_TELEMETRY = os.environ.get("CONDOR_TELEMETRY", "").strip().lower() or None -# Where a batch would be POSTed. Deliberately unset by default and NOT baked -# into the source: with no URL the send path is inert, and events can only ever -# accumulate in the local capped outbox. See PRIVACY.md. -CONDOR_TELEMETRY_URL = os.environ.get("CONDOR_TELEMETRY_URL", "").strip() or None +# ── Run mode (FEAT-049) ── +# Condor runs either as a Telegram bot (the default) or as a local, Telegram-less +# dashboard. The mode is *explicit*: it is read from CONDOR_MODE and is never +# inferred from whether a token happens to be present. An install whose +# TELEGRAM_TOKEN went missing must fail loudly at boot, not quietly become an +# unauthenticated dashboard — local mode has no authentication at all. + + +MODE_TELEGRAM = "telegram" +MODE_LOCAL = "local" + + +def resolve_mode(env=None) -> str: + """The configured run mode: ``"telegram"`` (default) or ``"local"``. + + Only the exact value ``local`` (case- and space-insensitive) selects local + mode. Anything else — unset, empty, or a typo — is ``telegram``, which is + the fail-safe direction: telegram mode requires a token, so a mistyped mode + surfaces as the hard error below instead of as an open dashboard. + """ + env = os.environ if env is None else env + value = (env.get("CONDOR_MODE") or "").strip().lower() + return MODE_LOCAL if value == MODE_LOCAL else MODE_TELEGRAM + + +def resolve_web_host(env=None) -> str: + """The address the dashboard binds to. + + Local mode has no login, so it binds loopback only: an unauthenticated + dashboard with full trading control must not be one firewall rule away from + the internet. ``WEB_HOST`` is the explicit, documented opt-out (set it to + ``0.0.0.0`` and you have chosen to expose it). Telegram mode, which does + authenticate, keeps binding all interfaces. + """ + env = os.environ if env is None else env + explicit = (env.get("WEB_HOST") or "").strip() + if explicit: + return explicit + return "127.0.0.1" if resolve_mode(env) == MODE_LOCAL else "0.0.0.0" + + +def resolve_local_user_id(env=None) -> int: + """The user id local mode logs in as: the admin, or ``1`` when there is none. + + Local mode deliberately introduces no new identity concept — not even its + own. It logs in as ``ADMIN_USER_ID``, the same id ``config.yml``'s + ``admin_id``, ``server_access.owner_id``, ``chat_defaults``, preferences, + memory and session keys already key on, and the one ``ConfigManager`` + materialises a user record for. Two knobs for one identity is what broke: + a Telegram install that flipped ``CONDOR_MODE=local`` kept its Telegram + ``ADMIN_USER_ID`` while the dashboard tried to log in as a hardcoded ``1`` + that ``config.yml`` had never heard of. + + ``1`` remains the answer for an install that never had a Telegram id — it is + what ``make setup`` writes as ``ADMIN_USER_ID`` in local mode — and ``1`` + rather than ``0`` because several call sites read a falsy id as "absent". + """ + return resolve_admin_id(env) or 1 + + +def check_startup_config(env=None) -> None: + """Refuse to start on a configuration that cannot mean what it says. + + Two cases, both of which used to surface far from their cause: + + * Telegram mode (including the default, i.e. an unset ``CONDOR_MODE``) with + no token. That surfaced deep inside PTB as an ``InvalidToken`` traceback; + more importantly it must *never* be treated as "well, local mode then". + * An ``ADMIN_USER_ID`` that is not a positive integer. That used to be + swallowed silently, leaving an install with **no admin at all**: no admin + panel, no approvals, no boot notification, and in local mode no user to + log in as — all of it looking like unrelated breakage later. + """ + env = os.environ if env is None else env + + raw_admin = (env.get("ADMIN_USER_ID") or "").strip() + if raw_admin and resolve_admin_id(env) is None: + raise ConfigError( + f"ADMIN_USER_ID is not a valid user id: {raw_admin!r}.\n" + "It must be a positive integer — your numeric Telegram user id " + "(get it from https://t.me/userinfobot), or 1 in local mode.\n" + "Fix it in .env, or run `make setup`." + ) + + if ( + resolve_mode(env) != MODE_LOCAL + and not (env.get("TELEGRAM_TOKEN") or "").strip() + ): + raise ConfigError( + "TELEGRAM_TOKEN is not set and CONDOR_MODE is not 'local'.\n" + "Run `make setup` to configure a Telegram bot, or choose Local mode " + "there (CONDOR_MODE=local) to run the dashboard without Telegram." + ) + + +def check_local_user(env=None, get_role=None) -> None: + """In local mode, refuse to start unless the user it logs in as exists. + + Local mode logs in without a password, so the *only* thing that decides who + that is happens before any request: ``ADMIN_USER_ID`` must resolve to a + ``config.yml`` user with a role that can use the dashboard. When it does + not, the failure belongs here — at boot, in ``make run``, next to the file + you have to edit — and not as a 500 from ``/auth/local-login`` once the + browser is already open. + + ``get_role`` is injectable so this is testable without a config file; it + defaults to the real :class:`ConfigManager`, whose load also materialises + the admin's user record from the environment. + """ + env = os.environ if env is None else env + if resolve_mode(env) != MODE_LOCAL: + return + + user_id = resolve_local_user_id(env) + if get_role is None: + from config_manager import get_config_manager + + get_role = get_config_manager().get_user_role + + from config_manager import UserRole + + if get_role(user_id) not in (UserRole.USER, UserRole.ADMIN): + raw_admin = (env.get("ADMIN_USER_ID") or "").strip() + source = ( + f"ADMIN_USER_ID={raw_admin} in .env" + if raw_admin + else "the local-mode default (ADMIN_USER_ID is not set in .env)" + ) + raise ConfigError( + f"Local mode logs in as user {user_id} — {source} — but that user " + f"is not an approved user in config.yml.\n" + "Run `make setup` and choose Local mode, or set ADMIN_USER_ID in " + ".env to a user that exists in config.yml.\n" + "Switching an existing Telegram install to local mode: keep your " + "Telegram ADMIN_USER_ID and the dashboard logs in as you, with all " + "your servers and preferences intact." + ) + + +CONDOR_MODE = resolve_mode() +LOCAL_MODE = CONDOR_MODE == MODE_LOCAL +WEB_HOST = resolve_web_host() +LOCAL_USER_ID = resolve_local_user_id()