From 6b2cb43c596c0087441205ae88db19f66f615441 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Tue, 18 Aug 2026 23:35:10 +0300 Subject: [PATCH 01/57] (feat) apply improvements requested by ralph # Conflicts: # frontend/src/pages/Dex.tsx # frontend/src/pages/DexPool.tsx --- condor/web/models.py | 3 + condor/web/routes/servers.py | 5 + condor/web/routes/settings.py | 6 + .../src/components/dex/PoolSourceTabs.tsx | 188 ++++++++---------- frontend/src/components/dex/PoolStats.tsx | 81 +++++--- .../src/components/executor/LPConfigPanel.tsx | 144 ++++++++------ .../components/executor/OrderConfigPanel.tsx | 7 +- frontend/src/components/executor/fields.tsx | 29 ++- frontend/src/components/executor/lp-config.ts | 37 +++- .../src/components/layout/ServerSelector.tsx | 24 ++- .../components/settings/ServersSettings.tsx | 33 ++- frontend/src/lib/api.ts | 2 + frontend/src/pages/Dex.tsx | 5 +- frontend/src/pages/DexPool.tsx | 23 ++- tests/test_web_servers_parallel_status.py | 34 +++- 15 files changed, 404 insertions(+), 217 deletions(-) diff --git a/condor/web/models.py b/condor/web/models.py index 757a51a24..553acf465 100644 --- a/condor/web/models.py +++ b/condor/web/models.py @@ -38,6 +38,9 @@ class ServerInfo(BaseModel): port: int online: bool = False permission: str = "trader" + # The server this user's commands land on when none is named — the same + # `chat_defaults` entry Telegram reads, so both surfaces agree on it. + is_default: bool = False # ── Portfolio ── diff --git a/condor/web/routes/servers.py b/condor/web/routes/servers.py index 9d94afd91..72d0645e1 100644 --- a/condor/web/routes/servers.py +++ b/condor/web/routes/servers.py @@ -47,6 +47,10 @@ async def list_servers(user: WebUser = Depends(get_current_user)): names = list(accessible) flags = await asyncio.gather(*(_resolve_online(sds, name) for name in names)) + # The selector seeds itself from this on a browser that has not picked a + # server yet, so a default set in Telegram is the one the dashboard opens on. + default_server = cm.get_chat_default_server(user.id) + results = [] for name, online in zip(names, flags): cfg = accessible[name] @@ -58,6 +62,7 @@ async def list_servers(user: WebUser = Depends(get_current_user)): port=cfg.get("port", 0), online=online, permission=perm.value if perm else "trader", + is_default=name == default_server, ) ) diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index d996f5610..b5b68dcfa 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -85,6 +85,11 @@ async def _get_status(name: str) -> dict: statuses = await asyncio.gather(*[_get_status(name) for name in accessible]) + # Which one the star should be filled on. Read once: the config manager falls + # back to the global default and then to the first server, so this is the + # server that actually answers, not only an explicit `chat_defaults` entry. + default_server = cm.get_chat_default_server(user.id) + results = [] for (name, cfg), status in zip(accessible.items(), statuses): perm = cm.get_server_permission(user.id, name) @@ -96,6 +101,7 @@ async def _get_status(name: str) -> dict: port=cfg.get("port", 0), online=online, permission=perm.value if perm else "trader", + is_default=name == default_server, ) ) diff --git a/frontend/src/components/dex/PoolSourceTabs.tsx b/frontend/src/components/dex/PoolSourceTabs.tsx index 16e8046d8..1bec4963a 100644 --- a/frontend/src/components/dex/PoolSourceTabs.tsx +++ b/frontend/src/components/dex/PoolSourceTabs.tsx @@ -1,6 +1,7 @@ import { Check, ChevronDown, Search, Star, X } from "lucide-react"; -import { useEffect, useMemo, useRef, useState } from "react"; +import { useCallback, useMemo, useState } from "react"; +import { AnchoredMenu } from "@/components/ui/AnchoredMenu"; import type { DexChain, DexVenue } from "@/lib/api"; import { @@ -45,22 +46,8 @@ function ChainSelect({ onChange: (next: string) => void; }) { const [open, setOpen] = useState(false); - const ref = useRef(null); - - useEffect(() => { - const handler = (e: MouseEvent) => { - if (ref.current && !ref.current.contains(e.target as Node)) setOpen(false); - }; - const keyHandler = (e: KeyboardEvent) => { - if (e.key === "Escape") setOpen(false); - }; - document.addEventListener("mousedown", handler); - document.addEventListener("keydown", keyHandler); - return () => { - document.removeEventListener("mousedown", handler); - document.removeEventListener("keydown", keyHandler); - }; - }, []); + const [anchor, setAnchor] = useState(null); + const close = useCallback(() => setOpen(false), []); // One chain is not a choice — render it as a label rather than a dead dropdown. if (chains.length <= 1) return null; @@ -68,8 +55,9 @@ function ChainSelect({ const current = chains.find((c) => c.network_id === network); return ( -
+ <> - {open && ( -
- {chains.map((c) => ( - - ))} -
- )} -
+ + {chains.map((c) => ( + + ))} + + ); } @@ -126,23 +112,8 @@ function DexFilter({ }) { const [open, setOpen] = useState(false); const [filter, setFilter] = useState(""); - const ref = useRef(null); - - useEffect(() => { - const handler = (e: MouseEvent) => { - if (ref.current && !ref.current.contains(e.target as Node)) - setOpen(false); - }; - const keyHandler = (e: KeyboardEvent) => { - if (e.key === "Escape") setOpen(false); - }; - document.addEventListener("mousedown", handler); - document.addEventListener("keydown", keyHandler); - return () => { - document.removeEventListener("mousedown", handler); - document.removeEventListener("keydown", keyHandler); - }; - }, []); + const [anchor, setAnchor] = useState(null); + const close = useCallback(() => setOpen(false), []); const shown = useMemo(() => { const needle = filter.trim().toLowerCase(); @@ -171,8 +142,9 @@ function DexFilter({ : `${selected.length} DEXes`; return ( -
+ <> - {open && ( -
-
- setFilter(e.target.value)} - placeholder="Filter venues…" - autoFocus - spellCheck={false} - className="w-full rounded-md border border-[var(--color-border)] bg-[var(--color-bg)] px-2 py-1 text-xs text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]" - /> -
-
- {shown.map((v) => { + +
+ setFilter(e.target.value)} + placeholder="Filter venues…" + autoFocus + spellCheck={false} + className="w-full rounded-md border border-[var(--color-border)] bg-[var(--color-bg)] px-2 py-1 text-xs text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]" + /> +
+
+ {shown.map((v) => { const active = selected.includes(v.id); - return ( - - ); - })} - {!shown.length && ( -
- No venue matches. -
- )} -
+ {active && } + + {v.name} + + ); + })} + {!shown.length && ( +
+ No venue matches. +
+ )}
- )} -
+ + ); } diff --git a/frontend/src/components/dex/PoolStats.tsx b/frontend/src/components/dex/PoolStats.tsx index f82aef4d0..f27cc0623 100644 --- a/frontend/src/components/dex/PoolStats.tsx +++ b/frontend/src/components/dex/PoolStats.tsx @@ -3,6 +3,17 @@ import { useState } from "react"; import type { PoolSummary } from "@/lib/api"; +/** GeckoTerminal chain ids → the slugs the DEX web apps route on. */ +const UNISWAP_CHAINS: Record = { + eth: "ethereum", + bsc: "bnb", + polygon_pos: "polygon", +}; + +const PANCAKE_CHAINS: Record = { + arbitrum: "arb", +}; + /** The DEX web app URL for a pool — the same mapping Telegram's /lp offers. */ function poolUrl(pool: PoolSummary): string { const dex = pool.dex_id.toLowerCase(); @@ -11,6 +22,16 @@ function poolUrl(pool: PoolSummary): string { return `https://app.meteora.ag/dlmm/${addr}?referrer=hummingbot`; if (dex.startsWith("raydium")) return `https://raydium.io/clmm/pool/${addr}`; if (dex.startsWith("orca")) return `https://www.orca.so/pools/${addr}`; + if (dex.startsWith("pumpswap") && pool.base_token_address) + return `https://pump.fun/coin/${pool.base_token_address}`; + if (dex.startsWith("uniswap")) { + const chain = UNISWAP_CHAINS[pool.gecko_network] ?? pool.gecko_network; + return `https://app.uniswap.org/explore/pools/${chain}/${addr}`; + } + if (dex.startsWith("pancakeswap")) { + const chain = PANCAKE_CHAINS[pool.gecko_network] ?? pool.gecko_network; + return `https://pancakeswap.finance/liquidity/pool/${chain}/${addr}`; + } // GeckoTerminal indexes every pool it listed, so it is the universal fallback. return `https://www.geckoterminal.com/${pool.gecko_network}/pools/${addr}`; } @@ -42,10 +63,9 @@ function Stat({ label, value, tone }: { label: string; value: string; tone?: str ); } -/** The pool's identity and market numbers, above the chart. */ -export function PoolStats({ pool }: { pool: PoolSummary }) { +/** The shortened pool address with copy and open-on-DEX actions. */ +export function PoolAddress({ pool }: { pool: PoolSummary }) { const [copied, setCopied] = useState(false); - const change = pool.price_change_24h; const copy = () => { navigator.clipboard.writeText(pool.address); @@ -53,8 +73,38 @@ export function PoolStats({ pool }: { pool: PoolSummary }) { setTimeout(() => setCopied(false), 1500); }; + return ( + + + {pool.address.slice(0, 6)}…{pool.address.slice(-4)} + + + + + + + ); +} + +/** The pool's identity and market numbers, above the chart. */ +export function PoolStats({ pool, network }: { pool: PoolSummary; network?: string }) { + const change = pool.price_change_24h; + return (
+ {network && } @@ -79,31 +129,6 @@ export function PoolStats({ pool }: { pool: PoolSummary }) { pool.base_fee_percentage !== undefined && ( )} - -
- - {pool.address.slice(0, 6)}…{pool.address.slice(-4)} - - - - - -
); } diff --git a/frontend/src/components/executor/LPConfigPanel.tsx b/frontend/src/components/executor/LPConfigPanel.tsx index 4c62c6fcb..a3a8b4a82 100644 --- a/frontend/src/components/executor/LPConfigPanel.tsx +++ b/frontend/src/components/executor/LPConfigPanel.tsx @@ -78,6 +78,13 @@ interface Props { pair?: string; pool?: DexPoolInfo; poolFetching?: boolean; + /** + * The pool's DLMM bin step in basis points, when the caller knows it. Only the + * pool workspace does — the resolve-by-pair endpoint the create page uses does + * not report one — and it is what turns a too-wide range into a warning here + * rather than a Gateway error at open time. + */ + binStep?: number | null; /** Wallet balance behind the percentage presets; `null` when it is unknown. */ baseAvailable?: number | null; quoteAvailable?: number | null; @@ -87,6 +94,12 @@ interface Props { */ baseSymbol?: string; quoteSymbol?: string; + /** + * Set when the page itself pins the pool (the DEX pool workspace). The panel + * then just states the address — resolution status, an editable address and a + * provider picker would only second-guess a decision already made. + */ + lockedPoolAddress?: string; } export function LPConfigPanel({ @@ -97,10 +110,12 @@ export function LPConfigPanel({ pair, pool, poolFetching, + binStep, baseAvailable = null, quoteAvailable = null, baseSymbol, quoteSymbol, + lockedPoolAddress, }: Props) { const d = dispatch as FieldDispatch; @@ -120,75 +135,86 @@ export function LPConfigPanel({ } }, [price, unanchored]); // eslint-disable-line react-hooks/exhaustive-deps - const warnings = rangeWarnings(state, price); + // Only DLMM pools carry a bin step, so passing it through unconditionally is + // enough to scope the bin-count warning to the pools the cap applies to. + const warnings = rangeWarnings(state, price, binStep); const unsupported = !!pool && !pool.lp_supported; return (
{/* Resolved pool */} -
- Pool - {poolFetching && !pool ? ( -

Resolving pool…

- ) : unsupported ? ( -
-

- - - The deepest pool for this pair is on{" "} - {pool?.dex_id ?? "an unknown venue"}, which - is not a CLMM venue. Enter a pool address and provider by hand. - -

-
- ) : pool?.pool_address ? ( -
- - {pool.dex_id} - - - {truncateAddress(pool.pool_address)} - - {pool.current_price != null && ( - - {formatPoolPrice(pool.current_price)} {quoteAsset} + {lockedPoolAddress ? ( +

+ {truncateAddress(lockedPoolAddress)} +

+ ) : ( +
+ Pool + {poolFetching && !pool ? ( +

Resolving pool…

+ ) : unsupported ? ( +
+

+ + + The deepest pool for this pair is on{" "} + {pool?.dex_id ?? "an unknown venue"}, which + is not a CLMM venue. Enter a pool address and provider by hand. + +

+
+ ) : pool?.pool_address ? ( +
+ + {pool.dex_id} - )} - {state.poolTouched && ( - - overridden + + {truncateAddress(pool.pool_address)} - )} -
- ) : ( -

- No pool found for this pair. Enter one by hand. -

- )} + {pool.current_price != null && ( + + {formatPoolPrice(pool.current_price)} {quoteAsset} + + )} + {state.poolTouched && ( + + overridden + + )} +
+ ) : ( +

+ No pool found for this pair. Enter one by hand. +

+ )} -
- - - d({ type: "SET_FIELD", field: "pool_address", value: e.target.value.trim() }) - } - placeholder="Pool contract address" - spellCheck={false} - className="w-full rounded border border-[var(--color-border)] bg-[var(--color-bg)] px-2.5 py-1.5 font-mono text-[11px] text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]/40 focus:border-[var(--color-primary)] focus:outline-none" +
+ + + d({ type: "SET_FIELD", field: "pool_address", value: e.target.value.trim() }) + } + placeholder="Pool contract address" + spellCheck={false} + className="w-full rounded border border-[var(--color-border)] bg-[var(--color-bg)] px-2.5 py-1.5 font-mono text-[11px] text-[var(--color-text)] placeholder:text-[var(--color-text-muted)]/40 focus:border-[var(--color-primary)] focus:outline-none" + /> +
+
- -
+ )} {/* Side */}
diff --git a/frontend/src/components/executor/OrderConfigPanel.tsx b/frontend/src/components/executor/OrderConfigPanel.tsx index 2a9b3aab4..d6c707dc4 100644 --- a/frontend/src/components/executor/OrderConfigPanel.tsx +++ b/frontend/src/components/executor/OrderConfigPanel.tsx @@ -251,8 +251,9 @@ export function OrderConfigPanel({ return (
- {/* Direction */} - + {/* Direction — Buy/Sell on spot: Long/Short would suggest a perp, and a + "short" without tokens to sell is not a thing a swap can do. */} + {/* Order Config */}
@@ -280,6 +281,8 @@ export function OrderConfigPanel({ field="execution_strategy" dispatch={d} options={options} + // A DEX only swaps at market; a one-option dropdown reads as broken. + disabled={options.length <= 1} /> {!isSpot && ( diff --git a/frontend/src/components/executor/fields.tsx b/frontend/src/components/executor/fields.tsx index d4d959c0b..d5a4c3f33 100644 --- a/frontend/src/components/executor/fields.tsx +++ b/frontend/src/components/executor/fields.tsx @@ -227,12 +227,15 @@ export function SelectField({ field, dispatch, options, + disabled = false, }: { label: string; value: number | string; field: string; dispatch: FieldDispatch; options: { value: number | string; label: string }[]; + /** A select with nothing to select reads as broken unless it says so. */ + disabled?: boolean; }) { const [open, setOpen] = useState(false); const containerRef = useRef(null); @@ -265,13 +268,20 @@ export function SelectField({ aria-label={label} aria-expanded={open} aria-haspopup="listbox" + disabled={disabled} onClick={() => setOpen(!open)} - className="flex w-full items-center justify-between rounded-lg border border-[var(--color-border)] bg-[var(--color-surface)] px-2.5 py-1.5 text-xs text-[var(--color-text)] transition-colors hover:bg-[var(--color-surface-hover)]" + className={`flex w-full items-center justify-between rounded-lg border border-[var(--color-border)] px-2.5 py-1.5 text-xs transition-colors ${ + disabled + ? "bg-[var(--color-bg)] text-[var(--color-text-muted)] opacity-60" + : "bg-[var(--color-surface)] text-[var(--color-text)] hover:bg-[var(--color-surface-hover)]" + }`} > {selected?.label} - + {!disabled && ( + + )} - {open && ( + {open && !disabled && (
{options.map((opt) => { const isActive = opt.value === value; @@ -533,7 +543,16 @@ export function LeverageField({ -export function SideSelector({ side, dispatch }: { side: 1 | 2; dispatch: FieldDispatch }) { +export function SideSelector({ + side, + dispatch, + isSpot = false, +}: { + side: 1 | 2; + dispatch: FieldDispatch; + /** Spot has no position to be long or short of — the sides are Buy and Sell. */ + isSpot?: boolean; +}) { return (
Direction @@ -549,7 +568,7 @@ export function SideSelector({ side, dispatch }: { side: 1 | 2; dispatch: FieldD }`} style={side === opt.value ? { backgroundColor: opt.color } : undefined} > - {opt.label} + {isSpot ? (opt.value === 1 ? "BUY" : "SELL") : opt.label} ))}
diff --git a/frontend/src/components/executor/lp-config.ts b/frontend/src/components/executor/lp-config.ts index 5b966af6c..f7324196f 100644 --- a/frontend/src/components/executor/lp-config.ts +++ b/frontend/src/components/executor/lp-config.ts @@ -228,12 +228,40 @@ export function useLpValidation(state: LPState): ExecutorValidation { }, [state]); } +/** + * Bins a Meteora DLMM position of this width would span. + * + * Each bin is a fixed geometric step of `1 + bin_step/10000`, so the count is the + * log of the price ratio over the log of that step. Null when the pool is not a + * DLMM (only those report a bin step) or the bounds are not a real range. + */ +export function dlmmBinSpan( + lower: number, + upper: number, + binStep: number | null | undefined, +): number | null { + if (!binStep || binStep <= 0) return null; + if (!(lower > 0) || !(upper > lower)) return null; + return Math.ceil(Math.log(upper / lower) / Math.log(1 + binStep / 10000)); +} + +/** + * What one Meteora DLMM position can hold. A wider range is not rejected by the + * form — Gateway is the authority and its own cap can move — but it fails at + * open time, and until now the only trace of that was a line in the gateway log. + */ +export const DLMM_MAX_BINS = 69; + /** * Things the schema permits but that are rarely meant. Kept apart from validation * because none of them should block a create — a deliberately odd position is * still a position. */ -export function rangeWarnings(state: LPState, price: number | null): string[] { +export function rangeWarnings( + state: LPState, + price: number | null, + binStep?: number | null, +): string[] { const warnings: string[] = []; const bothAmounts = state.base_amount > 0 && state.quote_amount > 0; @@ -257,6 +285,13 @@ export function rangeWarnings(state: LPState, price: number | null): string[] { if (state.lower_limit_price > 0 && state.lower_limit_price >= state.lower_price) { warnings.push("Lower limit sits inside the range"); } + + const bins = dlmmBinSpan(state.lower_price, state.upper_price, binStep); + if (bins !== null && bins > DLMM_MAX_BINS) { + warnings.push( + `Range spans ~${bins} bins; Meteora opens at most ${DLMM_MAX_BINS} per position — narrow it or the open will fail`, + ); + } return warnings; } diff --git a/frontend/src/components/layout/ServerSelector.tsx b/frontend/src/components/layout/ServerSelector.tsx index 84b384d69..6a9ee1256 100644 --- a/frontend/src/components/layout/ServerSelector.tsx +++ b/frontend/src/components/layout/ServerSelector.tsx @@ -1,5 +1,5 @@ import { useQuery } from "@tanstack/react-query"; -import { ChevronDown, Circle, Server } from "lucide-react"; +import { ChevronDown, Circle, Server, Star } from "lucide-react"; import { useEffect, useMemo, useRef, useState } from "react"; import { useServer } from "@/hooks/useServer"; @@ -27,12 +27,22 @@ export function ServerSelector() { [JSON.stringify(onlineServers.map((s) => s.name))], ); - // Auto-select first online server only when no server is saved yet + // The server the backend calls this user's default — the same `chat_defaults` + // entry Telegram sets. A browser with nothing saved opens on it, so setting a + // default in either surface is honoured by the other. + const defaultOnlineName = useMemo( + () => onlineServers.find((s) => s.is_default)?.name ?? null, + // eslint-disable-next-line react-hooks/exhaustive-deps + [JSON.stringify(onlineServers.map((s) => [s.name, s.is_default]))], + ); + + // Auto-select only when no server is saved yet: the user's default if it is + // up, otherwise the first server that is. useEffect(() => { if (!server && onlineServerNames.length > 0) { - setServer(onlineServerNames[0]); + setServer(defaultOnlineName ?? onlineServerNames[0]); } - }, [server, onlineServerNames, setServer]); + }, [server, onlineServerNames, defaultOnlineName, setServer]); // Close on outside click or Escape useEffect(() => { @@ -90,6 +100,12 @@ export function ServerSelector() { > {s.name} + {s.is_default && ( + + )} {s.name === server && ( Active )} diff --git a/frontend/src/components/settings/ServersSettings.tsx b/frontend/src/components/settings/ServersSettings.tsx index 7331e962b..5eea5f873 100644 --- a/frontend/src/components/settings/ServersSettings.tsx +++ b/frontend/src/components/settings/ServersSettings.tsx @@ -10,6 +10,7 @@ import { } from "lucide-react"; import { useState } from "react"; +import { useServer } from "@/hooks/useServer"; import { type ServerInfo, api } from "@/lib/api"; interface ServerForm { @@ -24,6 +25,7 @@ const EMPTY_FORM: ServerForm = { name: "", host: "", port: 8000, username: "", p export function ServersSettings() { const qc = useQueryClient(); + const { server: activeServer, setServer } = useServer(); const [adding, setAdding] = useState(false); const [editing, setEditing] = useState(null); const [form, setForm] = useState(EMPTY_FORM); @@ -54,9 +56,16 @@ export function ServersSettings() { onSuccess: () => { invalidate(); setConfirmDelete(null); }, }); + // Setting the default is two things at once, because the button reads as one: + // it writes the `chat_defaults` entry Telegram also uses, and it points this + // browser at that server. Without the second half the click had no visible + // effect at all — the star was never filled and the header never moved. const defaultMut = useMutation({ mutationFn: (name: string) => api.setDefaultServer(name), - onSuccess: invalidate, + onSuccess: (_data, name) => { + if (name !== activeServer) setServer(name); + invalidate(); + }, }); const startEdit = (s: ServerInfo) => { @@ -204,6 +213,11 @@ export function ServersSettings() { {s.permission} + {s.is_default && ( + + Default + + )}
{s.host}:{s.port} @@ -214,10 +228,21 @@ export function ServersSettings() {
{isOwner(s) && ( <> diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index dae2d617c..70dbf985b 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -22,6 +22,8 @@ export interface ServerInfo { port: number; online: boolean; permission: string; + /** The server commands fall back to when none is named — shared with Telegram. */ + is_default: boolean; } export interface BalanceItem { diff --git a/frontend/src/pages/Dex.tsx b/frontend/src/pages/Dex.tsx index dc13e9764..be973b29d 100644 --- a/frontend/src/pages/Dex.tsx +++ b/frontend/src/pages/Dex.tsx @@ -309,8 +309,9 @@ export function Dex() { )} - {/* The pasted-pool answer above already covers an empty search result — - showing "No pools found" under it would contradict what's right above it. */} + {/* A pasted address that resolved to a pool IS the result: an empty + token-search table saying "No pools found" under it would deny the + row right above. */} {!(pastedPool && !pools.length) && (
{pairLabel} - {network} + {/* The address identifies the pool the way the ticker never can, so it + sits with the name instead of hiding at the far right. */} +
- +
@@ -372,7 +379,7 @@ export function DexPool() {
Interval - {INTERVALS.map((iv) => ( + {DEX_INTERVALS.map((iv) => ( + + {/* Icon */} +
+
+ +
+
+ + {/* Title */} +

+ {title} +

+ {subtitle && ( +

+ {subtitle} +

+ )} + + {/* Details */} +
+
+ Executor ID +
+ + {executorId.slice(0, 12)}… + + +
+
+ {details} +
+ + {/* Action */} + +
+
+ ); +} + +/** + * Bottom-right error toast for executor create mutations. Rendered inside a + * relatively-positioned page container. + */ +export function ErrorToast({ message }: { message: string }) { + return ( +
+ {message} +
+ ); +} diff --git a/frontend/src/pages/CreateExecutor.tsx b/frontend/src/pages/CreateExecutor.tsx index e8b23b2c2..9ff766734 100644 --- a/frontend/src/pages/CreateExecutor.tsx +++ b/frontend/src/pages/CreateExecutor.tsx @@ -5,8 +5,6 @@ import { ArrowLeft, ArrowUpDown, BarChart3, - CheckCircle, - Copy, Droplets, Grid3X3, Layers, @@ -15,7 +13,6 @@ import { Rocket, Settings2, TrendingUp, - X, } from "lucide-react"; import { NoServerCard } from "@/components/NoServerCard"; @@ -26,6 +23,10 @@ import { MarketDepthPanel } from "@/components/market/MarketDepthPanel"; import { MarketsPanel } from "@/components/market/MarketsPanel"; import { TradeChart } from "@/components/trade/TradeChart"; import { GridConfigPanel, useGridValidation } from "@/components/grid/GridConfigPanel"; +import { + ErrorToast, + ExecutorSuccessModal, +} from "@/components/executor/ExecutorSuccessModal"; import { PositionConfigPanel, usePositionConfig } from "@/components/executor/PositionConfigPanel"; import { OrderConfigPanel, useOrderConfig } from "@/components/executor/OrderConfigPanel"; import { DCAConfigPanel, useDCAConfig } from "@/components/executor/DCAConfigPanel"; @@ -41,7 +42,6 @@ import { useRates } from "@/hooks/useRates"; import { useResizeDrag } from "@/hooks/useResizeDrag"; import { api } from "@/lib/api"; import { candleStore } from "@/lib/candle-store"; -import { copyText } from "@/lib/clipboard"; import { connectorCapabilities } from "@/lib/connector-capabilities"; import type { ExecutorType, PickSlot } from "@/components/executor/types"; import { @@ -755,46 +755,12 @@ export function CreateExecutor() { {/* Success modal */} {successInfo && ( -
-
- {/* Close button */} - - - {/* Icon */} -
-
- -
-
- - {/* Title */} -

- {TYPE_LABELS[successInfo.type]} Created -

-

- Successfully deployed on {successInfo.connector} -

- - {/* Details */} -
-
- Executor ID -
- {successInfo.id.slice(0, 12)}… - -
-
+
Pair {successInfo.pair} @@ -806,24 +772,16 @@ export function CreateExecutor() { ACTIVE
-
- - {/* Action */} - -
-
+ + } + primaryLabel="Continue Trading" + onClose={() => setSuccessInfo(null)} + /> )} {/* Error toast */} {createMutation.isError && ( -
- {(createMutation.error as Error).message} -
+ )}
); diff --git a/frontend/src/pages/CreateGridExecutor.tsx b/frontend/src/pages/CreateGridExecutor.tsx index d007ff5b1..2bff9a459 100644 --- a/frontend/src/pages/CreateGridExecutor.tsx +++ b/frontend/src/pages/CreateGridExecutor.tsx @@ -1,9 +1,13 @@ import { useMutation, useQuery } from "@tanstack/react-query"; import { useEffect, useMemo, useReducer, useState } from "react"; import { useNavigate } from "react-router-dom"; -import { ArrowLeft, CheckCircle, Loader2, Rocket } from "lucide-react"; +import { ArrowLeft, Loader2, Rocket } from "lucide-react"; import { NoServerCard } from "@/components/NoServerCard"; +import { + ErrorToast, + ExecutorSuccessModal, +} from "@/components/executor/ExecutorSuccessModal"; import { ExchangeSelector } from "@/components/market/ExchangeSelector"; import { PairSelector, useTradingRules } from "@/components/market/PairSelector"; import { PriceTicker } from "@/components/market/PriceTicker"; @@ -314,29 +318,19 @@ export function CreateGridExecutor() { {/* Success modal */} {successId && ( -
-
- -
-

Grid Executor Created

-

{successId}

-
-

Redirecting to executors...

- -
-
+ navigate("/executors")} + onClose={() => setSuccessId(null)} + /> )} {/* Error toast */} {createMutation.isError && ( -
- {(createMutation.error as Error).message} -
+ )}
); diff --git a/frontend/src/pages/DexPool.tsx b/frontend/src/pages/DexPool.tsx index ea64e0598..b2bd02265 100644 --- a/frontend/src/pages/DexPool.tsx +++ b/frontend/src/pages/DexPool.tsx @@ -2,11 +2,8 @@ import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; import { AlertTriangle, ArrowLeft, - CheckCircle, ChevronLeft, ChevronRight, - Copy, - X, } from "lucide-react"; import { useEffect, useMemo, useState } from "react"; import { useNavigate, useParams } from "react-router-dom"; @@ -16,6 +13,10 @@ import { LiquidityDepthColumn } from "@/components/dex/LiquidityDepthColumn"; import { LpPositionBar } from "@/components/dex/LpPositionBar"; import { PoolAddress, PoolStats } from "@/components/dex/PoolStats"; import { UpstreamNotice } from "@/components/dex/UpstreamNotice"; +import { + ErrorToast, + ExecutorSuccessModal, +} from "@/components/executor/ExecutorSuccessModal"; import { LPConfigPanel } from "@/components/executor/LPConfigPanel"; import { useLpConfig } from "@/components/executor/lp-config"; import { @@ -31,7 +32,6 @@ import { useResizeDrag } from "@/hooks/useResizeDrag"; import { useServer } from "@/hooks/useServer"; import { useCondorWebSocket } from "@/hooks/useWebSocket"; import { api } from "@/lib/api"; -import { copyText } from "@/lib/clipboard"; import { connectorCapabilities } from "@/lib/connector-capabilities"; import { LOOKBACK_OPTIONS } from "@/lib/gridExecutor"; @@ -629,52 +629,16 @@ export function DexPool() {
{successId && ( -
-
- -
-
- -
-
-

- {tab === "lp" ? "LP Position" : "Order"} Created -

-

- In {pairLabel} on {pool.dex_id} -

-
- Executor ID -
- {successId.slice(0, 12)}… - -
-
- -
-
+ setSuccessId(null)} + /> )} {createMutation.isError && ( -
- {(createMutation.error as Error).message} -
+ )}
); From 23611bfd15821a3bbcbbb480c201ac605094eb06 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 16:37:51 +0300 Subject: [PATCH 32/57] (fix) serialize concurrent session creation per key (CORR-187) get_or_create_session was an unserialized check-then-create spanning the subprocess spawn: two concurrent calls for one key both spawned a full client and the second registration overwrote the first, orphaning its process tree until shutdown. Creation now runs under a refcounted per-key lock (re-checking the registry after acquiring), and in-flight creations reserve their budget slot so concurrent creates of distinct keys cannot exceed MAX_SESSIONS_PER_USER. Tests race real concurrent creates for the orphan, budget, parallelism, and fast-path cases. --- condor/runtime/sessions.py | 108 ++++++++++- tests/runtime/test_session_creation_lock.py | 196 ++++++++++++++++++++ 2 files changed, 299 insertions(+), 5 deletions(-) create mode 100644 tests/runtime/test_session_creation_lock.py diff --git a/condor/runtime/sessions.py b/condor/runtime/sessions.py index 3d8543358..1858f3d50 100644 --- a/condor/runtime/sessions.py +++ b/condor/runtime/sessions.py @@ -11,6 +11,7 @@ import asyncio import logging +from contextlib import asynccontextmanager from dataclasses import dataclass, field from datetime import datetime, timezone @@ -51,6 +52,44 @@ # Not persisted -- subprocesses can't survive restarts. _sessions: dict[str, "AgentSession"] = {} +# Creation serialization (CORR-187). get_or_create_session is a check-then- +# create that spans many awaits (subprocess spawn, eager context prompt), so +# without a lock two concurrent calls for the same key both spawn a full +# client and the second registration overwrites the first, whose process tree +# is never stopped -- teardown only walks the registry. Locks are per session +# key (creations under different keys stay concurrent) and refcounted so the +# dict does not grow with every key ever seen. +_creation_locks: dict[str, asyncio.Lock] = {} +_creation_lock_refs: dict[str, int] = {} + +# Session keys currently being created, per user. A creation in flight is not +# in ``_sessions`` yet but will register one session, so the budget check has +# to count it or N concurrent creates of distinct keys all pass the cap. +_pending_creates: dict[int, set[str]] = {} + + +@asynccontextmanager +async def _creation_lock(name: str): + """Hold the named creation lock; drop it when the last holder leaves. + + Refcounted rather than popped on release: popping while another task still + waits on the lock object would hand a *fresh* lock to the next caller and + reopen the race between the waiter and the newcomer. + """ + lock = _creation_locks.setdefault(name, asyncio.Lock()) + _creation_lock_refs[name] = _creation_lock_refs.get(name, 0) + 1 + try: + async with lock: + yield + finally: + remaining = _creation_lock_refs[name] - 1 + if remaining: + _creation_lock_refs[name] = remaining + else: + del _creation_lock_refs[name] + _creation_locks.pop(name, None) + + # Health monitor state _health_task: asyncio.Task | None = None _health_bot: Bot | None = None @@ -244,7 +283,11 @@ async def _enforce_session_budget(user_id: int) -> None: while True: mine = [s for s in _sessions.values() if s.user_id == user_id] - if len(mine) < MAX_SESSIONS_PER_USER: + # Creations in flight for this user (reserved under the per-user + # creation lock, registered only later) hold budget too, or N + # concurrent creates of distinct keys would all pass the count. + in_flight = len(_pending_creates.get(user_id, set()) - set(_sessions)) + if len(mine) + in_flight < MAX_SESSIONS_PER_USER: return idle = [s for s in mine if not s.is_busy] @@ -357,6 +400,25 @@ async def get_or_create_session( """ key = SessionKey.parse(spec.key) raw_key = str(key) + + # One creation at a time per key (CORR-187): the second concurrent caller + # waits here and then finds the first caller's session in the registry + # instead of spawning a duplicate whose twin would leak until shutdown. + # Locks are per key, so creations under different keys stay concurrent. + async with _creation_lock(raw_key): + return await _get_or_create_session_locked( + spec, key, raw_key, permission_callback, user_data + ) + + +async def _get_or_create_session_locked( + spec: SessionSpec, + key: SessionKey, + raw_key: str, + permission_callback: PermissionCallback | None, + user_data: dict | None, +) -> AgentSession: + """Body of :func:`get_or_create_session`; runs under the per-key lock.""" session = _sessions.get(raw_key) # Reuse existing session only if the same brain is still on the other end: @@ -384,11 +446,47 @@ async def get_or_create_session( # Destroy old session if exists if session: await _destroy_session_internal(key) - elif spec.user_id: - # Only a genuinely new key counts against the budget — replacing the - # session behind an existing key is a swap, not an extra subprocess. - await _enforce_session_budget(spec.user_id) + if spec.user_id: + if session is None: + # Only a genuinely new key counts against the budget — replacing + # the session behind an existing key is a swap, not an extra + # subprocess. + await _enforce_session_budget(spec.user_id) + # Reserve this key's budget slot for the whole spawn. The reservation + # must follow the check with no await in between — the event loop + # makes the pair atomic, so two creates can never both pass the count + # before either reserves (and _enforce_session_budget recounts after + # every await it does make). The swap path reserves too: its old + # session is already out of the registry, so a concurrent create + # would otherwise undercount during the respawn window. + _pending_creates.setdefault(spec.user_id, set()).add(raw_key) + try: + return await _spawn_session( + spec, key, raw_key, permission_callback, user_data + ) + finally: + reserved = _pending_creates.get(spec.user_id) + if reserved is not None: + reserved.discard(raw_key) + if not reserved: + del _pending_creates[spec.user_id] + + return await _spawn_session(spec, key, raw_key, permission_callback, user_data) + + +async def _spawn_session( + spec: SessionSpec, + key: SessionKey, + raw_key: str, + permission_callback: PermissionCallback | None, + user_data: dict | None, +) -> AgentSession: + """Spawn, contextualize, and register a new session for ``raw_key``. + + Runs under the per-key creation lock, with the key's budget slot already + reserved when the spec names a user. + """ # MCP subprocess env expects a numeric chat id; surfaces without a chat # (web, mcp) fall back to the user id. The same pair goes down on argv via # binding.resolve, which is what the subprocess actually reads first — so diff --git a/tests/runtime/test_session_creation_lock.py b/tests/runtime/test_session_creation_lock.py new file mode 100644 index 000000000..7d71bfe65 --- /dev/null +++ b/tests/runtime/test_session_creation_lock.py @@ -0,0 +1,196 @@ +"""Concurrent session creation is serialized per key (CORR-187). + +``get_or_create_session`` is a check-then-create that spans many awaits +(subprocess spawn, eager context prompt). Before the per-key creation lock, +two concurrent calls for the same key both spawned a full client and the +second registration overwrote the first, whose process tree leaked until bot +shutdown; the session budget had the same TOCTOU. These tests *race* real +concurrent calls — they do not merely assert that a lock object exists. +""" + +import asyncio + +import pytest + +from condor.runtime import SessionKey, SessionSpec +from condor.runtime import sessions as session_module + + +class _SlowClient: + """Client stub whose start() yields to the loop, opening the race window.""" + + instances: list = [] + started = 0 + stopped = 0 + # When set, start() blocks until the event fires (for the parallelism test). + start_gate: asyncio.Event | None = None + + def __init__(self, **kwargs): + self.alive = True + type(self).instances.append(self) + + async def start(self): + type(self).started += 1 + if type(self).start_gate is not None: + await type(self).start_gate.wait() + else: + # Yield a few times so a concurrent caller can reach the registry + # check while this creation is mid-flight. + for _ in range(3): + await asyncio.sleep(0) + + async def stop(self): + self.alive = False + type(self).stopped += 1 + + async def prompt(self, text): + return "" + + async def prompt_stream(self, text): + return + yield # pragma: no cover - makes this an async generator + + +@pytest.fixture +def registry(monkeypatch): + """Fresh module state with the subprocess client stubbed out.""" + _SlowClient.instances = [] + _SlowClient.started = 0 + _SlowClient.stopped = 0 + _SlowClient.start_gate = None + monkeypatch.setattr(session_module, "_sessions", {}) + monkeypatch.setattr(session_module, "_pending_creates", {}) + monkeypatch.setattr(session_module, "_creation_locks", {}) + monkeypatch.setattr(session_module, "_creation_lock_refs", {}) + monkeypatch.setattr(session_module, "ACPClient", _SlowClient) + monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") + monkeypatch.setattr( + "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + ) + return session_module + + +def _spec(key: SessionKey, user_id: int = 42) -> SessionSpec: + return SessionSpec(key=str(key), agent_key="claude-code", user_id=user_id) + + +def _assert_no_orphans(registry): + """Every started client is either registered in _sessions or stopped.""" + registered = {s.client for s in registry._sessions.values()} + for client in _SlowClient.instances: + assert ( + client in registered or client.stopped or not client.alive + ), "orphaned client: started but neither registered nor stopped" + + +def test_concurrent_same_key_spawns_once(registry): + """Two racing creates for one key: one start(), one shared session.""" + key = SessionKey.web(42, "slot1") + + async def scenario(): + return await asyncio.gather( + session_module.get_or_create_session(_spec(key)), + session_module.get_or_create_session(_spec(key)), + ) + + first, second = asyncio.run(scenario()) + assert first is second + assert _SlowClient.started == 1 + assert len(registry._sessions) == 1 + assert registry._sessions[str(key)] is first + + +def test_racing_creates_leave_no_orphan(registry): + """N racing creates for one key never strand a started client.""" + key = SessionKey.web(42, "slot1") + + async def scenario(): + return await asyncio.gather( + *(session_module.get_or_create_session(_spec(key)) for _ in range(5)) + ) + + results = asyncio.run(scenario()) + assert len({id(s) for s in results}) == 1 + _assert_no_orphans(registry) + assert _SlowClient.started == 1 + + +def test_budget_holds_under_concurrent_distinct_creates(registry): + """Racing creates of distinct keys never exceed MAX_SESSIONS_PER_USER.""" + cap = session_module.MAX_SESSIONS_PER_USER + total = cap + 3 + + async def scenario(): + return await asyncio.gather( + *( + session_module.get_or_create_session( + _spec(SessionKey.web(42, f"slot{i}")) + ) + for i in range(total) + ), + return_exceptions=True, + ) + + results = asyncio.run(scenario()) + sessions = [r for r in results if isinstance(r, session_module.AgentSession)] + refused = [r for r in results if isinstance(r, session_module.SessionLimitReached)] + unexpected = [ + r + for r in results + if not isinstance( + r, (session_module.AgentSession, session_module.SessionLimitReached) + ) + ] + assert not unexpected + assert len(sessions) == cap + assert len(refused) == total - cap + assert len([s for s in registry._sessions.values() if s.user_id == 42]) == cap + # The refused creates never spawned anything, so nothing to orphan. + assert _SlowClient.started == cap + _assert_no_orphans(registry) + + +def test_distinct_keys_create_concurrently(registry): + """Per-key locks: creations under different keys are not serialized.""" + _SlowClient.start_gate = asyncio.Event() # replaced inside the loop + + async def scenario(): + gate = asyncio.Event() + _SlowClient.start_gate = gate + tasks = [ + asyncio.create_task( + session_module.get_or_create_session( + _spec(SessionKey.web(42, f"slot{i}")) + ) + ) + for i in range(2) + ] + # Both creations must reach start() while neither has finished — + # a global creation lock would leave the second waiting forever. + async with asyncio.timeout(2): + while _SlowClient.started < 2: + await asyncio.sleep(0) + gate.set() + return await asyncio.gather(*tasks) + + first, second = asyncio.run(scenario()) + assert first is not second + assert len(registry._sessions) == 2 + + +def test_fast_path_returns_without_spawning(registry): + """An existing matching session is returned with no new spawn.""" + key = SessionKey.web(42, "slot1") + + async def scenario(): + created = await session_module.get_or_create_session(_spec(key)) + again = await session_module.get_or_create_session(_spec(key)) + return created, again + + created, again = asyncio.run(scenario()) + assert created is again + assert _SlowClient.started == 1 + # The reservation bookkeeping is fully cleaned up after every create. + assert registry._pending_creates == {} + assert registry._creation_locks == {} + assert registry._creation_lock_refs == {} From 17b6a128a01cf95a7c988ded6c271a19fd9e2871 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 16:44:29 +0300 Subject: [PATCH 33/57] (perf) cache instance-history cursor walks with TTL + in-flight coalescing --- condor/fetchers/bot_performance.py | 111 +++++++++++++++--- tests/test_bot_performance.py | 158 +++++++++++++++++++++++++- tests/test_session_pnl_attribution.py | 7 +- 3 files changed, 258 insertions(+), 18 deletions(-) diff --git a/condor/fetchers/bot_performance.py b/condor/fetchers/bot_performance.py index cdfcad6f4..e9f0e32ee 100644 --- a/condor/fetchers/bot_performance.py +++ b/condor/fetchers/bot_performance.py @@ -20,7 +20,7 @@ import asyncio import logging import time -from collections import Counter +from collections import Counter, OrderedDict from functools import partial from typing import Any, Iterable @@ -475,6 +475,32 @@ def extract_history_rows(result: Any) -> list[dict]: HISTORY_PAGE_SIZE = 500 +# ── Per-instance history cache ── +# The cursor walk is the expensive end of the pipeline — up to 10 sequential +# 500-row pages per instance, fanned out over up to MAX_HISTORY_INSTANCES +# instances — and every consumer walks the same rows: one session-detail request +# triggers it twice back-to-back (performance rollup, then PnL series), and the +# tick engine repeats it every tick while the dashboard polls the same strategy. +# The rows are identical across those call sites because the fetch takes no +# since/until — window slicing happens downstream — so a short TTL plus in-flight +# coalescing (the _snapshot_cache idiom above) collapses the redundancy while +# staying fresher than the 30s route-level cache layered on top. Bounded as an +# LRU: keys accumulate one entry per (server, instance, interval) and stopped +# instances would otherwise pin their pages forever. +_HISTORY_TTL = 20.0 +_HISTORY_CACHE_MAX = 256 +_history_cache: OrderedDict[ + tuple, tuple[float, list[tuple[float, float, float, float, float]]] +] = OrderedDict() +_history_inflight: dict[tuple, tuple[Any, asyncio.Task]] = {} + + +def clear_history_cache() -> None: + """Drop every cached instance history (tests, server reconfiguration).""" + _history_cache.clear() + _history_inflight.clear() + + async def fetch_instance_history( client: Any, instance_name: str, @@ -521,7 +547,64 @@ async def fetch_instance_history( error, including one raised part-way through the walk — a partial timeline is the same silent misattribution as a truncated one, so the caller degrades to "no history" instead of to "wrong history". + + Cached per ``(server, instance, interval, limit, max_rows)`` for + ``_HISTORY_TTL`` seconds and coalesced while in flight, so concurrent callers + share one cursor walk. A walk that raises is never cached — every waiter gets + the ``[]`` degrade and the next call retries — and an unidentifiable client + (no ``base_url``) bypasses the cache entirely, like the snapshot cache above. + The returned rows are shared between callers and must be treated as + read-only. """ + server = _server_key(client) + if not server: + try: + return await _walk_instance_history( + client, instance_name, interval, limit, max_rows + ) + except Exception as e: + logger.debug("fetch_instance_history(%s) failed: %s", instance_name, e) + return [] + + key = (server, instance_name, interval, limit, max_rows) + entry = _history_cache.get(key) + if entry is not None and time.monotonic() - entry[0] <= _HISTORY_TTL: + _history_cache.move_to_end(key) + return entry[1] + + # Reuse an in-flight walk only from the loop that created it: a task is + # bound to its loop and awaiting it from another one raises. + loop = asyncio.get_running_loop() + inflight = _history_inflight.get(key) + task = inflight[1] if inflight is not None and inflight[0] is loop else None + if task is None: + task = asyncio.ensure_future( + _walk_instance_history(client, instance_name, interval, limit, max_rows) + ) + _history_inflight[key] = (loop, task) + task.add_done_callback(lambda _t, k=key: _history_inflight.pop(k, None)) + + try: + rows = await task + except Exception as e: + logger.debug("fetch_instance_history(%s) failed: %s", instance_name, e) + return [] + + _history_cache[key] = (time.monotonic(), rows) + _history_cache.move_to_end(key) + while len(_history_cache) > _HISTORY_CACHE_MAX: + _history_cache.popitem(last=False) + return rows + + +async def _walk_instance_history( + client: Any, + instance_name: str, + interval: str, + limit: int, + max_rows: int, +) -> list[tuple[float, float, float, float, float]]: + """One uncached cursor walk + forward-carry merge. Raises on API error.""" def _warn_truncated() -> None: # Older buckets were dropped, and everything before the oldest retained @@ -535,21 +618,17 @@ def _warn_truncated() -> None: interval, ) - try: - rows: list[dict] = await collect_pages( - partial( - client.bot_orchestration.get_controller_performance_history, - bot_name=instance_name, - interval=interval, - ), - extract_history_rows, - page_size=limit, - max_items=max_rows, - on_truncated=_warn_truncated, - ) - except Exception as e: - logger.debug("fetch_instance_history(%s) failed: %s", instance_name, e) - return [] + rows: list[dict] = await collect_pages( + partial( + client.bot_orchestration.get_controller_performance_history, + bot_name=instance_name, + interval=interval, + ), + extract_history_rows, + page_size=limit, + max_items=max_rows, + on_truncated=_warn_truncated, + ) # One cumulative series per controller. A repeated (controller, timestamp) # keeps the last value read, which is what the endpoint means by re-reporting diff --git a/tests/test_bot_performance.py b/tests/test_bot_performance.py index 18e03f8a2..b0bd3d53f 100644 --- a/tests/test_bot_performance.py +++ b/tests/test_bot_performance.py @@ -19,6 +19,7 @@ from condor.fetchers.bot_performance import ( _aggregate_by_bot, bot_executor_rows, + clear_history_cache, clear_snapshot_cache, extract_snapshots, fetch_all_bot_performance, @@ -27,10 +28,12 @@ @pytest.fixture(autouse=True) def _no_snapshot_cache_bleed(): - """Isolate the per-server snapshot cache between tests.""" + """Isolate the per-server snapshot and history caches between tests.""" clear_snapshot_cache() + clear_history_cache() yield clear_snapshot_cache() + clear_history_cache() # ── Sample payloads ── @@ -1034,6 +1037,159 @@ def test_history_forward_carry_survives_a_controller_that_stops_reporting(): assert [round(r[2], 4) for r in hist] == [100.0, 100.0, 100.0] +# ── PERF-184: instance-history cursor walks are cached + coalesced ── +# +# One session-detail request walks the same instance's history twice +# back-to-back (performance rollup, then PnL series), and the tick engine +# repeats the walk every tick while the dashboard polls the same strategy. The +# cache collapses those to one walk per (server, instance, interval) per TTL. + + +class _CountingHistory: + def __init__(self, rows, delay=0.0, fail_first=False): + self.rows = rows + self.delay = delay + self.fail_first = fail_first + self.calls = 0 + + async def get_controller_performance_history(self, **_kw): + self.calls += 1 + if self.delay: + await asyncio.sleep(self.delay) + if self.fail_first and self.calls == 1: + raise RuntimeError("api down") + return {"data": self.rows} + + +def _history_server_client(base_url, rows, **kw): + return SimpleNamespace( + base_url=base_url, bot_orchestration=_CountingHistory(rows, **kw) + ) + + +_HIST_ROWS = [ + _hist_row("2026-08-06T22:00:00+00:00", "btc", 1.0, 10.0), + _hist_row("2026-08-06T22:15:00+00:00", "btc", 2.0, 20.0), +] + + +def test_repeated_history_reads_within_ttl_share_one_walk(): + """The two walks inside one session-detail request collapse to one.""" + from condor.fetchers.bot_performance import fetch_instance_history + + client = _history_server_client("http://hist-a:8000", _HIST_ROWS) + + async def _go(): + first = await fetch_instance_history(client, "bot-1") + second = await fetch_instance_history(client, "bot-1") + return first, second + + first, second = asyncio.run(_go()) + assert client.bot_orchestration.calls == 1 + assert first == second + assert [r[1] for r in first] == [1.0, 2.0] + + +def test_concurrent_history_callers_share_one_inflight_walk(): + """Engine tick + dashboard poll arriving together join one cursor walk.""" + from condor.fetchers.bot_performance import fetch_instance_history + + client = _history_server_client("http://hist-a:8000", _HIST_ROWS, delay=0.02) + + async def _go(): + return await asyncio.gather( + *[fetch_instance_history(client, "bot-1") for _ in range(5)] + ) + + results = asyncio.run(_go()) + assert client.bot_orchestration.calls == 1 + assert all(len(r) == 2 for r in results) + + +def test_distinct_instances_and_intervals_never_share_a_walk(): + from condor.fetchers.bot_performance import fetch_instance_history + + client = _history_server_client("http://hist-a:8000", _HIST_ROWS) + + async def _go(): + await fetch_instance_history(client, "bot-1", interval="5m") + await fetch_instance_history(client, "bot-2", interval="5m") + await fetch_instance_history(client, "bot-1", interval="15m") + + asyncio.run(_go()) + assert client.bot_orchestration.calls == 3 + + +def test_failed_history_walk_is_not_cached(): + """The [] degrade on API error stays retryable — waiters share the failure.""" + from condor.fetchers.bot_performance import fetch_instance_history + + client = _history_server_client( + "http://flaky-hist:8000", _HIST_ROWS, delay=0.02, fail_first=True + ) + + async def _go(): + # Both waiters join the failing walk and both degrade to []. + failed = await asyncio.gather( + *[fetch_instance_history(client, "bot-1") for _ in range(2)] + ) + # Nothing was cached, so the next call really re-walks and succeeds. + recovered = await fetch_instance_history(client, "bot-1") + return failed, recovered + + failed, recovered = asyncio.run(_go()) + assert failed == [[], []] + assert client.bot_orchestration.calls == 2 + assert [r[1] for r in recovered] == [1.0, 2.0] + + +def test_history_client_without_base_url_bypasses_the_cache(): + from condor.fetchers.bot_performance import fetch_instance_history + + client = _history_server_client("", _HIST_ROWS) + + async def _go(): + await fetch_instance_history(client, "bot-1") + await fetch_instance_history(client, "bot-1") + + asyncio.run(_go()) + assert client.bot_orchestration.calls == 2 + + +def test_history_cache_is_bounded_lru(monkeypatch): + """Stopped instances must not pin their pages forever.""" + import condor.fetchers.bot_performance as bp + + monkeypatch.setattr(bp, "_HISTORY_CACHE_MAX", 2) + client = _history_server_client("http://hist-a:8000", _HIST_ROWS) + + async def _go(): + await bp.fetch_instance_history(client, "bot-1") + await bp.fetch_instance_history(client, "bot-2") + await bp.fetch_instance_history(client, "bot-3") # evicts bot-1 + assert len(bp._history_cache) == 2 + await bp.fetch_instance_history(client, "bot-1") # miss: re-walks + + asyncio.run(_go()) + assert client.bot_orchestration.calls == 4 + + +def test_history_inflight_from_another_loop_is_not_awaited(): + """A task created on one event loop is never awaited from another.""" + import condor.fetchers.bot_performance as bp + + client = _history_server_client("http://hist-a:8000", _HIST_ROWS) + key = ("http://hist-a:8000", "bot-1", "5m", 500, 5000) + # A stale in-flight entry left by a different loop: the guard must ignore it + # (awaiting a foreign loop's task raises) and start its own walk. + foreign = SimpleNamespace() # never awaited — would blow up if it were + bp._history_inflight[key] = (object(), foreign) + + hist = asyncio.run(bp.fetch_instance_history(client, "bot-1")) + assert client.bot_orchestration.calls == 1 + assert [r[1] for r in hist] == [1.0, 2.0] + + def test_partition_includes_archived_instances(): """A stopped bot's realized PnL is exactly what its session is judged on.""" from condor.fetchers.bot_performance import partition_instances diff --git a/tests/test_session_pnl_attribution.py b/tests/test_session_pnl_attribution.py index caa5e458a..bbb80df34 100644 --- a/tests/test_session_pnl_attribution.py +++ b/tests/test_session_pnl_attribution.py @@ -462,7 +462,10 @@ def test_rollup_fans_out_one_snapshot_call_for_all_strategies(tmp_path): ``get_latest_controller_performance()`` returns the same whole-server payload to each of them — so the fan-out must collapse to one round-trip. """ - from condor.fetchers.bot_performance import clear_snapshot_cache + from condor.fetchers.bot_performance import ( + clear_history_cache, + clear_snapshot_cache, + ) class _CountingClient(_FakeClient): def __init__(self, *args, **kwargs): @@ -495,10 +498,12 @@ async def _go(): ) clear_snapshot_cache() + clear_history_cache() try: asyncio.run(_go()) finally: clear_snapshot_cache() + clear_history_cache() assert client.snapshot_calls == 1 # Every strategy still gets its attribution from that one snapshot. From 5ad0e4618a729813f4cb284c4e533abefd0b2a5a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 16:50:33 +0300 Subject: [PATCH 34/57] (refactor) consolidate frontmatter round-trip and slugify into condor/frontmatter (ARCH-202) One implementation of parse_frontmatter/render_frontmatter/slugify replaces the drifted private copies in agents/strategy.py and memory/store.py. agent.py now parses AND renders AGENT.md through the same module; the stricter YAML-warning parse behavior is kept, and the memory/skill stores keep their historical "memory" slug fallback via a thin binding. --- condor/agents/README.md | 2 +- condor/agents/agent.py | 10 +++--- condor/agents/ownership.py | 2 +- condor/agents/shutdown.py | 5 +-- condor/agents/strategy.py | 53 +++-------------------------- condor/frontmatter.py | 68 +++++++++++++++++++++++++++++++++++++ condor/memory/skills.py | 16 +++++---- condor/memory/store.py | 53 +++++++---------------------- tests/test_bot_ownership.py | 6 ++-- tests/test_memory_store.py | 4 +-- tests/test_skill_store.py | 4 +-- 11 files changed, 110 insertions(+), 113 deletions(-) create mode 100644 condor/frontmatter.py diff --git a/condor/agents/README.md b/condor/agents/README.md index 33e7ca69e..9a132acc1 100644 --- a/condor/agents/README.md +++ b/condor/agents/README.md @@ -151,7 +151,7 @@ The name is resolved once at engine start and written back into the session's `c - The **base rules at the top of the prompt switch with it** (`BASE_PROMPT_LIVE_CONTROLLER` vs `BASE_PROMPT_LIVE_EXECUTORS` in `condor/agents/prompts.py`): controller mode names controllers as the trading surface, demotes `manage_executors` to an opt-in fallback, preloads `manage_bots` + `manage_controllers` via ToolSearch, and drops the executor-only `controller_id` hint. A single shared base prompt used to open every tick with "Trade ONLY via `manage_executors`" and then contradict itself in the `[CONTROLLER MODE]` block. - The agent's reported PnL becomes `executor_pnl(controller_id == agent_id)` **+** `bot_pnl(bot_name == config.bot_name)`. The two sources are disjoint — bot controllers tag their executors with their own config ids, never the `agent_id` — so the merge is plain addition with no double counting (`condor/fetchers/bot_performance.py` aggregates the bot side; `condor/agents/performance.py` folds it in). -**Ownership is a namespace, enforced at the tool call** (FEAT-017, `condor/agents/ownership.py`). Every bot a strategy may deploy or mutate lives under `{agent_slug}-{strategy_slug}` — the base itself, any tagged sibling (`…-btc`, `…-eth`), and their deployed instances (Hummingbot appends a `-YYYYMMDD-HHMMSS` timestamp). Slugs never contain `-` (`strategy._slugify` maps `[\s-]+ → _`), so the delimiter is unambiguous and two agents can never claim the same bot — uniqueness is structural, with no registry or lock. +**Ownership is a namespace, enforced at the tool call** (FEAT-017, `condor/agents/ownership.py`). Every bot a strategy may deploy or mutate lives under `{agent_slug}-{strategy_slug}` — the base itself, any tagged sibling (`…-btc`, `…-eth`), and their deployed instances (Hummingbot appends a `-YYYYMMDD-HHMMSS` timestamp). Slugs never contain `-` (`condor.frontmatter.slugify` maps `[\s-]+ → _`), so the delimiter is unambiguous and two agents can never claim the same bot — uniqueness is structural, with no registry or lock. - A `manage_bots` call with a *dangerous* action (`deploy`, `stop_bot`, `start_controllers`, `stop_controllers`, `update_config`) on a bot outside the namespace is **cancelled** at the permission callback (`condor/agents/risk.py:auto_approve_with_risk_check`) and recorded. Read-only actions (`status`, `logs`, `get_config`) are never restricted — an agent still sees the whole fleet. - A `bot_name` configured before this convention (e.g. `river-scalper`) is carried as a **declared** name and stays owned, so existing agents keep working with no config change. diff --git a/condor/agents/agent.py b/condor/agents/agent.py index 4b0b1d9df..e72a0bbcd 100644 --- a/condor/agents/agent.py +++ b/condor/agents/agent.py @@ -46,11 +46,9 @@ from datetime import datetime, timezone from pathlib import Path +from condor.frontmatter import parse_frontmatter, render_frontmatter, slugify from condor.fsutil import atomic_write_text from condor.memory.paths import CHAT_SLUG -from condor.memory.store import _parse_frontmatter - -from .strategy import _render_frontmatter, _slugify log = logging.getLogger(__name__) @@ -161,7 +159,7 @@ def _load_agent_from_dir(agent_dir: Path) -> Agent | None: if not path.exists(): return None try: - meta, body = _parse_frontmatter(path.read_text()) + meta, body = parse_frontmatter(path.read_text()) return Agent( slug=agent_dir.name, name=meta.get("name", agent_dir.name), @@ -255,7 +253,7 @@ def create( server_name: str = "", created_by: int = 0, ) -> Agent: - slug = _slugify(name) + slug = slugify(name) if slug == CHAT_SLUG: # Reserved: `condor` names the default agent. The separate trees used # to make the collision impossible by construction (FEAT-003); with @@ -319,7 +317,7 @@ def _save(self, agent: Agent) -> None: agent.agent_dir.mkdir(parents=True, exist_ok=True) atomic_write_text( agent.agent_dir / "AGENT.md", - _render_frontmatter(meta, agent.instructions), + render_frontmatter(meta, agent.instructions), ) def _iter_agent_dirs(self): diff --git a/condor/agents/ownership.py b/condor/agents/ownership.py index c1a289064..1ffb33427 100644 --- a/condor/agents/ownership.py +++ b/condor/agents/ownership.py @@ -3,7 +3,7 @@ An agent running in controller mode owns the bots it deploys, and the **name is the proof**: every bot a given ``(agent, strategy)`` may touch lives under the namespace ``{agent_slug}-{strategy_slug}``. Slugs never contain ``-`` -(``strategy._slugify`` maps ``[\\s-]+ → _``), so ``-`` delimits the namespace +(``condor.frontmatter.slugify`` maps ``[\\s-]+ → _``), so ``-`` delimits the namespace unambiguously and two different agents can never claim the same bot — uniqueness is structural, with no registry, lock or lookup. diff --git a/condor/agents/shutdown.py b/condor/agents/shutdown.py index a6ed6095a..03f8c8886 100644 --- a/condor/agents/shutdown.py +++ b/condor/agents/shutdown.py @@ -20,9 +20,10 @@ import logging from typing import Any +from condor.frontmatter import parse_frontmatter from condor.runtime.timeouts import resolve_tick_timeout -from .strategy import Strategy, _parse_frontmatter +from .strategy import Strategy log = logging.getLogger(__name__) @@ -93,7 +94,7 @@ def load_shutdown_policy(strategy: Strategy) -> tuple[ShutdownPolicy, str]: if not path.exists(): continue try: - meta, body = _parse_frontmatter(path.read_text()) + meta, body = parse_frontmatter(path.read_text()) return ShutdownPolicy.from_dict(meta), body.strip() except Exception: log.exception("Failed to parse shutdown.md at %s", path) diff --git a/condor/agents/strategy.py b/condor/agents/strategy.py index 28f7d0d06..80efc94de 100644 --- a/condor/agents/strategy.py +++ b/condor/agents/strategy.py @@ -27,15 +27,13 @@ from __future__ import annotations import logging -import re import shutil from dataclasses import dataclass, field from datetime import datetime, timezone from pathlib import Path from typing import Any -import yaml - +from condor.frontmatter import parse_frontmatter, render_frontmatter, slugify from condor.fsutil import atomic_write_text log = logging.getLogger(__name__) @@ -64,47 +62,6 @@ """ -def _slugify(name: str) -> str: - """Convert a name to a filesystem-safe slug. - - Example: "RIVER Scalper v2" -> "river_scalper_v2" - """ - s = name.lower().strip() - s = re.sub(r"[^\w\s-]", "", s) - s = re.sub(r"[\s-]+", "_", s) - return s.strip("_") or "unnamed" - - -def _parse_frontmatter(text: str) -> tuple[dict, str]: - """Parse YAML frontmatter and markdown body from a file.""" - text = text.strip() - if not text.startswith("---"): - return {}, text - - end = text.find("---", 3) - if end == -1: - return {}, text - - frontmatter_str = text[3:end].strip() - body = text[end + 3 :].strip() - - try: - meta = yaml.safe_load(frontmatter_str) or {} - except yaml.YAMLError: - log.warning("Failed to parse YAML frontmatter") - meta = {} - - return meta, body - - -def _render_frontmatter(meta: dict, body: str) -> str: - """Render YAML frontmatter + markdown body.""" - frontmatter = yaml.dump( - meta, default_flow_style=False, allow_unicode=True, sort_keys=False - ).strip() - return f"---\n{frontmatter}\n---\n\n{body}\n" - - @dataclass class Strategy: agent_slug: str # the owning Agent's slug @@ -125,7 +82,7 @@ def __post_init__(self): @property def slug(self) -> str: """Filesystem-safe slug derived from the strategy name (unique per agent).""" - return _slugify(self.name) + return slugify(self.name) @property def key(self) -> str: @@ -141,7 +98,7 @@ def dir(self) -> Path: def split_key(key: str) -> tuple[str, str] | None: """Split an opaque strategy key ``"{agent_slug}.{slug}"`` into its parts. - Slugs never contain ``.`` (``_slugify`` strips it), so the first dot is the + Slugs never contain ``.`` (:func:`condor.frontmatter.slugify` strips it), so the first dot is the boundary. Returns None when the key has no dot. """ if "." not in key: @@ -153,7 +110,7 @@ def split_key(key: str) -> tuple[str, str] | None: def _load_strategy_from_file(path: Path, agent_slug: str) -> Strategy | None: """Load a Strategy from a ``strategy.md`` file under an agent.""" try: - meta, body = _parse_frontmatter(path.read_text()) + meta, body = parse_frontmatter(path.read_text()) return Strategy( agent_slug=agent_slug, name=meta.get("name", path.parent.name), @@ -328,5 +285,5 @@ def _save(self, strategy: Strategy) -> None: strategy.dir.mkdir(parents=True, exist_ok=True) atomic_write_text( self._strategy_md_path(strategy), - _render_frontmatter(meta, strategy.instructions), + render_frontmatter(meta, strategy.instructions), ) diff --git a/condor/frontmatter.py b/condor/frontmatter.py new file mode 100644 index 000000000..5e47ea34b --- /dev/null +++ b/condor/frontmatter.py @@ -0,0 +1,68 @@ +"""YAML-frontmatter round-trip and slug helpers — the one implementation. + +Every markdown-with-frontmatter file Condor persists (``AGENT.md``, +``strategy.md``, memory facts, skill playbooks) goes through this module, so +parse and render can never drift apart between callers. Consolidated from the +near-verbatim private copies in ``condor/agents/strategy.py`` and +``condor/memory/store.py`` (ARCH-202), following the same pattern that +collapsed the atomic-write copies into :mod:`condor.fsutil` (ARCH-148). + +The rendered format is byte-identical to what both former copies produced +(``yaml.dump`` with ``default_flow_style=False``, ``allow_unicode=True``, +``sort_keys=False``), so existing on-disk files round-trip unchanged. +""" + +from __future__ import annotations + +import logging +import re + +import yaml + +log = logging.getLogger(__name__) + + +def slugify(name: str, fallback: str = "unnamed") -> str: + """Convert a name to a filesystem-safe slug. + + Example: "RIVER Scalper v2" -> "river_scalper_v2" + + ``fallback`` is returned when nothing sluggable remains (empty or + all-punctuation names); callers pass their historical word so existing + on-disk slugs are preserved (agents/strategies use the default + ``"unnamed"``, the memory/skill stores use ``"memory"``). + """ + s = name.lower().strip() + s = re.sub(r"[^\w\s-]", "", s) + s = re.sub(r"[\s-]+", "_", s) + return s.strip("_") or fallback + + +def parse_frontmatter(text: str) -> tuple[dict, str]: + """Parse YAML frontmatter and markdown body from a file.""" + text = text.strip() + if not text.startswith("---"): + return {}, text + + end = text.find("---", 3) + if end == -1: + return {}, text + + frontmatter_str = text[3:end].strip() + body = text[end + 3 :].strip() + + try: + meta = yaml.safe_load(frontmatter_str) or {} + except yaml.YAMLError: + log.warning("Failed to parse YAML frontmatter") + meta = {} + + return meta, body + + +def render_frontmatter(meta: dict, body: str) -> str: + """Render YAML frontmatter + markdown body.""" + frontmatter = yaml.dump( + meta, default_flow_style=False, allow_unicode=True, sort_keys=False + ).strip() + return f"---\n{frontmatter}\n---\n\n{body}\n" diff --git a/condor/memory/skills.py b/condor/memory/skills.py index be0a7817d..cf79ec886 100644 --- a/condor/memory/skills.py +++ b/condor/memory/skills.py @@ -61,8 +61,10 @@ import shutil from pathlib import Path +from condor.frontmatter import parse_frontmatter, render_frontmatter + from .paths import builtin_skills_root, shared_skills_root -from .store import _atomic_write, _parse_frontmatter, _render, _slugify, _utcnow +from .store import _atomic_write, _slugify, _utcnow def _routine_exists(name: str, agent_slug: str | None = None) -> bool: @@ -197,7 +199,7 @@ def create( # Preserve the original created date on overwrite. created = _utcnow() if path.exists(): - existing_meta, _ = _parse_frontmatter(path.read_text()) + existing_meta, _ = parse_frontmatter(path.read_text()) created = existing_meta.get("created", created) meta = { @@ -211,7 +213,7 @@ def create( if ref: meta["references_routine"] = ref - _atomic_write(path, _render(meta, body.strip())) + _atomic_write(path, render_frontmatter(meta, body.strip())) result = { "saved": True, "name": slug, @@ -247,7 +249,7 @@ def edit(self, name: str, **fields) -> dict: if not path.exists(): return {"error": f"Skill '{name}' not found"} - meta, body = _parse_frontmatter(path.read_text()) + meta, body = parse_frontmatter(path.read_text()) if fields.get("description"): meta["description"] = fields["description"].strip().replace("\n", " ") if fields.get("when_to_use"): @@ -264,7 +266,7 @@ def edit(self, name: str, **fields) -> dict: # `_target_dir` above already moved the folder if it changed. meta.pop("shared", None) - _atomic_write(path, _render(meta, body)) + _atomic_write(path, render_frontmatter(meta, body)) return self.read(slug) or {"saved": True, "name": slug} def delete(self, name: str) -> bool | dict: @@ -310,7 +312,7 @@ def read(self, name: str) -> dict | None: if skill_dir is None: return None path = skill_dir / "SKILL.md" - meta, body = _parse_frontmatter(path.read_text()) + meta, body = parse_frontmatter(path.read_text()) ref = meta.get("references_routine") result = { "name": meta.get("name", slug), @@ -496,7 +498,7 @@ def _iter_skills(self): if slug in seen: continue try: - meta, body = _parse_frontmatter(f.read_text()) + meta, body = parse_frontmatter(f.read_text()) except Exception: continue seen.add(slug) diff --git a/condor/memory/store.py b/condor/memory/store.py index 0284824eb..e820482a6 100644 --- a/condor/memory/store.py +++ b/condor/memory/store.py @@ -22,12 +22,10 @@ from __future__ import annotations import json -import re from datetime import datetime, timezone from pathlib import Path -import yaml - +from condor.frontmatter import parse_frontmatter, render_frontmatter, slugify from condor.fsutil import atomic_write_text from .paths import store_root @@ -41,40 +39,13 @@ def _slugify(name: str) -> str: - """Convert a memory name to a filesystem-safe slug.""" - s = name.lower().strip() - s = re.sub(r"[^\w\s-]", "", s) - s = re.sub(r"[\s-]+", "_", s) - return s.strip("_") or "memory" - - -def _parse_frontmatter(text: str) -> tuple[dict, str]: - """Parse YAML frontmatter and markdown body (mirrors strategy.py).""" - text = text.strip() - if not text.startswith("---"): - return {}, text - - end = text.find("---", 3) - if end == -1: - return {}, text - - frontmatter_str = text[3:end].strip() - body = text[end + 3 :].strip() + """Slug with this store's historical ``"memory"`` fallback. - try: - meta = yaml.safe_load(frontmatter_str) or {} - except yaml.YAMLError: - meta = {} - - return meta, body - - -def _render(meta: dict, body: str) -> str: - """Render YAML frontmatter + markdown body.""" - frontmatter = yaml.dump( - meta, default_flow_style=False, allow_unicode=True, sort_keys=False - ).strip() - return f"---\n{frontmatter}\n---\n\n{body}\n" + Thin binding over :func:`condor.frontmatter.slugify` (ARCH-202) so + memories and skills alike keep resolving empty/all-punctuation names to + the same on-disk slug they always have. + """ + return slugify(name, fallback="memory") def _utcnow() -> str: @@ -187,7 +158,7 @@ def write( existed = path.exists() created = _utcnow() if existed: - existing_meta, _ = _parse_frontmatter(path.read_text()) + existing_meta, _ = parse_frontmatter(path.read_text()) created = existing_meta.get("created", created) meta = { @@ -197,7 +168,7 @@ def write( "created": created, "source": source, } - _atomic_write(path, _render(meta, content.strip())) + _atomic_write(path, render_frontmatter(meta, content.strip())) self._reindex() action = "update" if existed else "create" self._append_audit(action, f"memory:{slug}", meta["description"], source) @@ -213,7 +184,7 @@ def read(self, name: str) -> str | None: path = self.memories_dir / f"{_slugify(name)}.md" if not path.exists(): return None - _, body = _parse_frontmatter(path.read_text()) + _, body = parse_frontmatter(path.read_text()) return body def search(self, query: str, limit: int = 10) -> list[dict]: @@ -261,7 +232,7 @@ def delete(self, name: str, source: str = "user") -> bool: path = self.memories_dir / f"{slug}.md" if not path.exists(): return False - meta, _ = _parse_frontmatter(path.read_text()) + meta, _ = parse_frontmatter(path.read_text()) path.unlink() self._reindex() self._append_audit( @@ -295,7 +266,7 @@ def _iter_memories(self): parsed = [] for f in files: try: - meta, body = _parse_frontmatter(f.read_text()) + meta, body = parse_frontmatter(f.read_text()) except Exception: continue meta.setdefault("name", f.stem) diff --git a/tests/test_bot_ownership.py b/tests/test_bot_ownership.py index 861032a68..0eb35f04a 100644 --- a/tests/test_bot_ownership.py +++ b/tests/test_bot_ownership.py @@ -23,7 +23,7 @@ RiskState, auto_approve_with_risk_check, ) -from condor.agents.strategy import _slugify +from condor.frontmatter import slugify _OPTIONS = [{"kind": "allow_once", "optionId": "allow"}] NS = "brigado-ema_trend" @@ -50,9 +50,9 @@ def _drive(callback, call) -> str: def test_slugs_never_contain_the_delimiter(): - """The whole scheme rests on '-' being unambiguous: _slugify maps it to '_'.""" + """The whole scheme rests on '-' being unambiguous: slugify maps it to '_'.""" for name in ("RIVER Scalper v2", "ema-trend", " spot - perp ", "BTC/USD mm"): - assert "-" not in _slugify(name) + assert "-" not in slugify(name) def test_namespace_matches_base_tag_and_deployed_instance(): diff --git a/tests/test_memory_store.py b/tests/test_memory_store.py index ef9f38c08..421b06cc6 100644 --- a/tests/test_memory_store.py +++ b/tests/test_memory_store.py @@ -218,7 +218,7 @@ def test_concurrent_writers_never_leave_a_torn_file(memory_root): # always parse cleanly (frontmatter + body), never a torn/interleaved one. import threading - from condor.memory.store import _parse_frontmatter + from condor.frontmatter import parse_frontmatter s = MemoryStore(user_id=42) target = s.memories_dir / "shared.md" @@ -240,7 +240,7 @@ def writer(text): for t in threads: t.join() - meta, body = _parse_frontmatter(target.read_text()) + meta, body = parse_frontmatter(target.read_text()) # The surviving file is exactly one writer's payload, not a blend. assert meta.get("name") == "shared" assert body == "x" * 5000 diff --git a/tests/test_skill_store.py b/tests/test_skill_store.py index 8d6cf27f4..5bcaaa4b0 100644 --- a/tests/test_skill_store.py +++ b/tests/test_skill_store.py @@ -586,7 +586,7 @@ def test_concurrent_writers_never_leave_a_torn_file(project_root): # always parse cleanly, never a torn/interleaved one. import threading - from condor.memory.store import _parse_frontmatter + from condor.frontmatter import parse_frontmatter s = SkillStore() target = s.skills_dir / "shared" / "SKILL.md" @@ -608,7 +608,7 @@ def writer(text): for t in threads: t.join() - meta, body = _parse_frontmatter(target.read_text()) + meta, body = parse_frontmatter(target.read_text()) assert meta.get("name") == "shared" assert body == "x" * 5000 assert list(target.parent.glob("*.tmp")) == [] From aa27ec43eab3a31f775576e654857c5e53417e16 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 17:00:55 +0300 Subject: [PATCH 35/57] (fix) scope reports API to the report's owner (SEC-196) Reports were authenticated but not authorized: any approved user could list every report, read every HTML body (portfolio value, PnL, positions) and delete any report by id. Every runner (routine store, code runner, Telegram routine handler, agent session report) now stamps the authenticated principal on the index entry via reports.attribute_owner, the list endpoints filter by it, and the id-addressed endpoints follow the sessions idiom: own passes, admins see everything, and legacy entries with no owner are admin-only (fail closed), never world-readable. --- condor/agents/engine.py | 1 + condor/agents/session_report.py | 8 +- condor/code_runner.py | 27 +++- condor/reports/__init__.py | 2 + condor/reports/builder.py | 5 + condor/reports/store.py | 41 +++++- condor/routine_store.py | 9 +- condor/web/models.py | 3 + condor/web/routes/reports.py | 46 ++++++- handlers/routines/__init__.py | 13 +- tests/test_reports_ownership.py | 223 +++++++++++++++++++++++++++++++ tests/test_web_report_serving.py | 17 +++ 12 files changed, 378 insertions(+), 17 deletions(-) create mode 100644 tests/test_reports_ownership.py diff --git a/condor/agents/engine.py b/condor/agents/engine.py index 5be524798..acebc0e13 100644 --- a/condor/agents/engine.py +++ b/condor/agents/engine.py @@ -204,6 +204,7 @@ def __post_init__(self): self.strategy.slug, self.session_num, frequency_sec=self.config.get("frequency_sec", 60), + owner_id=self.user_id, ) risk_limits = RiskLimits.from_dict(self.config.get("risk_limits", {})) diff --git a/condor/agents/session_report.py b/condor/agents/session_report.py index 1fda39daa..8c2d64fb5 100644 --- a/condor/agents/session_report.py +++ b/condor/agents/session_report.py @@ -28,6 +28,7 @@ from pathlib import Path from typing import Any +from condor.reports import store from condor.reports.builder import LiveReport from . import canvas as canvas_mod @@ -47,9 +48,13 @@ def __init__( strategy_slug: str, session_num: int, frequency_sec: int = 60, + owner_id: int = 0, ) -> None: self.run_key = f"{agent_slug}.{strategy_slug}" self.session_num = session_num + # The engine's authenticated user_id: stamped on the report at save so + # the web reports routes can authorize reads against it (SEC-196). + self._owner_id = owner_id self._live = LiveReport( title=f"{agent_slug} · {strategy_slug} · session {session_num}", source_name=f"{self.run_key}/session_{session_num}", @@ -86,7 +91,8 @@ async def update( self._decisions(b, journal) self._belief_history(b, session_dir) - return await self._live.update() + with store.attribute_owner(self._owner_id): + return await self._live.update() # ------------------------------------------------------------------ # Blocks diff --git a/condor/code_runner.py b/condor/code_runner.py index 752843c85..0e28298be 100644 --- a/condor/code_runner.py +++ b/condor/code_runner.py @@ -124,18 +124,31 @@ def _render_result(namespace: dict) -> str: async def _run_in_task( - compiled, namespace: dict, buffer, agent: str, source: str, out: dict + compiled, + namespace: dict, + buffer, + agent: str, + source: str, + out: dict, + owner_id: int = 0, ): """The snippet itself, in its own task context. Setting the capture ContextVar *here* (not in the caller) is what keeps the buffer private to this run, and the report id is read in a ``finally`` so a snippet that saved a report and then raised still reports it. + ``owner_id`` is the authenticated caller (``chat_id`` in ``execute_code``); + it stamps any report the snippet saves so the web routes can authorize + reads/deletes against it (SEC-196). """ _CAPTURE.set(buffer) reports.reset_last_report_id() try: - with reports.attribute_to(agent), reports.default_source("code", source): + with ( + reports.attribute_owner(owner_id), + reports.attribute_to(agent), + reports.default_source("code", source), + ): outcome = eval(compiled, namespace) # noqa: S307 - this IS the feature if inspect.isawaitable(outcome): await outcome @@ -207,7 +220,15 @@ async def execute_code( _install_stdout_proxy() out: dict = {"report_id": ""} task = asyncio.create_task( - _run_in_task(compiled, namespace, buffer, agent, label or run_id, out) + _run_in_task( + compiled, + namespace, + buffer, + agent, + label or run_id, + out, + owner_id=chat_id, + ) ) try: await asyncio.wait_for(task, timeout=timeout) diff --git a/condor/reports/__init__.py b/condor/reports/__init__.py index 31fa4d3e9..e8001c7c1 100644 --- a/condor/reports/__init__.py +++ b/condor/reports/__init__.py @@ -5,6 +5,7 @@ CHARTS_DIR, INDEX_FILE, MAX_REPORTS, + attribute_owner, attribute_to, default_source, delete_report, @@ -22,6 +23,7 @@ "CHARTS_DIR", "INDEX_FILE", "MAX_REPORTS", + "attribute_owner", "attribute_to", "default_source", "reset_last_report_id", diff --git a/condor/reports/builder.py b/condor/reports/builder.py index 7220b4687..cabb84ff8 100644 --- a/condor/reports/builder.py +++ b/condor/reports/builder.py @@ -460,6 +460,11 @@ async def save(self, report_id: str | None = None) -> str: "source_name": source_name, "tags": self._tags, "agent": store._report_agent.get() or "condor", + # The authenticated principal the run executes for (SEC-196): + # every runner wraps execution in store.attribute_owner, and the + # web routes authorize reads/deletes against this id. None (no + # wrapper, e.g. a bare script) makes the report admin-only. + "user_id": store._report_owner.get(), } entries = store._read_index() entries.append(entry) diff --git a/condor/reports/store.py b/condor/reports/store.py index 7d00e2956..f796a7b66 100644 --- a/condor/reports/store.py +++ b/condor/reports/store.py @@ -26,6 +26,9 @@ _report_source: contextvars.ContextVar[tuple[str, str] | None] = contextvars.ContextVar( "report_source", default=None ) +_report_owner: contextvars.ContextVar[int | None] = contextvars.ContextVar( + "report_owner", default=None +) def _charts_dir() -> Path: @@ -62,6 +65,23 @@ def attribute_to(agent: str | None): _report_agent.reset(token) +@contextmanager +def attribute_owner(user_id: int | None): + """Attribute reports saved within this block to an authenticated user id. + + Every runner (routine store, code runner, Telegram routine handler, agent + session report) wraps execution with this so ``ReportBuilder.save`` records + who the report belongs to — the id the web routes then authorize reads and + deletes against (SEC-196). A falsy id (0, None) records no owner, which the + routes treat as admin-only, never world-readable. + """ + token = _report_owner.set(int(user_id) if user_id else None) + try: + yield + finally: + _report_owner.reset(token) + + @contextmanager def default_source(source_type: str, source_name: str): """Stamp reports saved in this block that never called ``ReportBuilder.source()``. @@ -128,10 +148,21 @@ def list_reports( agent: str | None = None, limit: int = 50, offset: int = 0, + owner_id: int | None = None, ) -> tuple[list[dict], int]: + """List index entries, newest first. + + ``owner_id`` scopes the listing to one user's reports: entries whose + ``user_id`` differs — including legacy entries with no owner at all — + are dropped (fail closed, SEC-196). ``None`` means no owner filter, + which the web routes reserve for admins; internal callers that match + on ``source_name`` (routine run lists) also pass ``None``. + """ entries = _read_index() entries.sort(key=lambda entry: entry.get("created_at", ""), reverse=True) + if owner_id is not None: + entries = [entry for entry in entries if entry.get("user_id") == owner_id] if source_type: entries = [ entry for entry in entries if entry.get("source_type") == source_type @@ -154,10 +185,16 @@ def list_reports( return entries[offset : offset + limit], total -def list_reports_grouped() -> list[dict]: - """Return the latest report per source name, with count.""" +def list_reports_grouped(owner_id: int | None = None) -> list[dict]: + """Return the latest report per source name, with count. + + ``owner_id`` scopes the grouping exactly as in ``list_reports``: ``None`` + (admin) sees every entry, anyone else only entries stamped with their id. + """ entries = _read_index() entries.sort(key=lambda entry: entry.get("created_at", ""), reverse=True) + if owner_id is not None: + entries = [entry for entry in entries if entry.get("user_id") == owner_id] groups: dict[str, dict] = {} for entry in entries: source_name = entry.get("source_name", "") diff --git a/condor/routine_store.py b/condor/routine_store.py index eba5b2db9..23aeb2cdf 100644 --- a/condor/routine_store.py +++ b/condor/routine_store.py @@ -552,9 +552,12 @@ async def _execute_and_record( # The bare name (not "agent_slug/name") is what both report lookups # match on, and what routines that do call .source() already use. base_name = (routine.name or "").split("/")[-1] - with reports.attribute_to(agent or _agent_of(routine)): - with reports.default_source("routine", base_name): - raw = await routine.run_fn(cfg, ctx) + # attribute_owner records who this run executes for, so its reports + # are readable by (and only by) that user on the web (SEC-196). + with reports.attribute_owner(user_id): + with reports.attribute_to(agent or _agent_of(routine)): + with reports.default_source("routine", base_name): + raw = await routine.run_fn(cfg, ctx) result = normalize_result(raw) except asyncio.CancelledError: result = RoutineResult(text="Stopped by user") diff --git a/condor/web/models.py b/condor/web/models.py index d77d3ad73..5d84cf2ce 100644 --- a/condor/web/models.py +++ b/condor/web/models.py @@ -461,6 +461,9 @@ class ReportSummary(BaseModel): source_name: str = "" tags: list[str] = [] agent: str = "" # producing assistant/expert (e.g. "condor", "executor_manager") + # Authenticated owner stamped at save time (SEC-196); None = legacy/ownerless, + # visible to admins only. + user_id: int | None = None class ReportsListResponse(BaseModel): diff --git a/condor/web/routes/reports.py b/condor/web/routes/reports.py index af3ed597b..98e01f35a 100644 --- a/condor/web/routes/reports.py +++ b/condor/web/routes/reports.py @@ -14,10 +14,42 @@ ) from condor.web.auth import get_current_user from condor.web.models import ReportsListResponse, ReportSummary, WebUser +from config_manager import get_config_manager router = APIRouter(prefix="/reports", tags=["reports"]) +def _owner_filter(user: WebUser) -> int | None: + """Whose reports a listing may show: everyone's for admins, own otherwise. + + ``None`` disables the store's owner filter — the admin override the other + server-data surfaces already grant (``_owner`` in conversations, + ``_require_ownership`` in sessions). Anyone else is scoped to entries + stamped with their own id; legacy entries with no owner are dropped for + them (fail closed, SEC-196). + """ + return None if get_config_manager().is_admin(user.id) else user.id + + +def _authorized_entry(report_id: str, user: WebUser) -> dict: + """Fetch an index entry, 404 if absent and 403 if it belongs to someone else. + + Ownership is the ``user_id`` the producer stamped at save time. Admins see + everything; an entry with no owner (written before SEC-196, or by a bare + script outside every runner) is admin-only rather than world-readable — + the same fail-closed rule ownerless routine instances follow. + """ + entry = get_report(report_id) + if not entry: + raise HTTPException(404, "Report not found") + if get_config_manager().is_admin(user.id): + return entry + owner = entry.get("user_id") + if owner is not None and owner == user.id: + return entry + raise HTTPException(status_code=403, detail="Not your report") + + @router.get("", response_model=ReportsListResponse) async def get_reports( source_type: str | None = None, @@ -35,6 +67,7 @@ async def get_reports( agent=agent, limit=limit, offset=offset, + owner_id=_owner_filter(user), ) return ReportsListResponse( reports=[ReportSummary(**e) for e in entries], @@ -44,28 +77,28 @@ async def get_reports( @router.get("/latest-by-source") async def get_reports_grouped(user: WebUser = Depends(get_current_user)): - return list_reports_grouped() + return list_reports_grouped(owner_id=_owner_filter(user)) @router.get("/{report_id}", response_model=ReportSummary) async def get_report_detail(report_id: str, user: WebUser = Depends(get_current_user)): - entry = get_report(report_id) - if not entry: - raise HTTPException(404, "Report not found") + entry = _authorized_entry(report_id, user) return ReportSummary(**entry) @router.get("/{report_id}/html", response_class=HTMLResponse) async def get_report_html(report_id: str, user: WebUser = Depends(get_current_user)): - """Return a report's rendered HTML body to an authenticated caller. + """Return a report's rendered HTML body to its owner (or an admin). Report bodies carry portfolio value, PnL, positions and strategy internals, so they are served from behind ``get_current_user`` like every other reports - surface — there is no unauthenticated static mount for them (SEC-112). The + surface — there is no unauthenticated static mount for them (SEC-112) — and, + since SEC-196, only to the report's recorded owner or an admin. The client fetches this with an ``Authorization`` header and hands the result to a sandboxed iframe via ``srcDoc``, which is why the body is returned inline rather than as a file the browser navigates to. """ + _authorized_entry(report_id, user) result = get_report_raw_html(report_id) if result is None: raise HTTPException(404, "Report not found") @@ -77,6 +110,7 @@ async def get_report_html(report_id: str, user: WebUser = Depends(get_current_us async def delete_report_endpoint( report_id: str, user: WebUser = Depends(get_current_user) ): + _authorized_entry(report_id, user) if not await delete_report(report_id): raise HTTPException(404, "Report not found") return {"deleted": True} diff --git a/handlers/routines/__init__.py b/handlers/routines/__init__.py index 41609f31b..5e441e6de 100644 --- a/handlers/routines/__init__.py +++ b/handlers/routines/__init__.py @@ -320,7 +320,13 @@ async def _execute_routine( try: config = routine.config_class(**config_dict) - raw_result = await routine.run_fn(config, context) + # Reports belong to the user whose user_data the run executes against + # (the starter), falling back to the chat — the same identity the hooks + # record (SEC-152) — so the web can authorize reads by it (SEC-196). + with condor.reports.attribute_owner( + owner_id if owner_id is not None else chat_id + ): + raw_result = await routine.run_fn(config, context) rich_result = normalize_result(raw_result) result_text = rich_result.text[:500] if rich_result.text else "Completed" except Exception as e: @@ -420,7 +426,10 @@ def user_data(self): try: config = routine.config_class(**config_dict) logger.info(f"Starting continuous routine {routine_name}[{instance_id}]") - result = await routine.run_fn(config, context) + # Same owner stamping as one-shot runs (SEC-196): reports this loop + # saves belong to the user the instance runs for. + with condor.reports.attribute_owner(owner_key): + result = await routine.run_fn(config, context) logger.info(f"Continuous routine {routine_name}[{instance_id}] ended: {result}") except asyncio.CancelledError: logger.info(f"Continuous routine {routine_name}[{instance_id}] cancelled") diff --git a/tests/test_reports_ownership.py b/tests/test_reports_ownership.py new file mode 100644 index 000000000..bb0fba0e0 --- /dev/null +++ b/tests/test_reports_ownership.py @@ -0,0 +1,223 @@ +"""SEC-196: reports are owned, and the API authorizes reads/deletes by owner. + +The reports index was the one server-data surface in condor/web not scoped per +user: any approved caller could list every report, read every HTML body (which +carries portfolio value, PnL and positions) and delete any of them. Now every +runner stamps the authenticated principal on the entry at save time, the list +endpoints filter by it, and the id-addressed endpoints follow the sessions +idiom: own passes, admins see everything, and legacy entries with no owner are +admin-only (fail closed), never world-readable. +""" + +import asyncio +import json + +import pytest +from starlette.testclient import TestClient + +import condor.reports as rep +import condor.web.routes.reports as routes +from condor.web.app import create_app +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +USER = WebUser(id=111, username="u", first_name="U", role="user") +OTHER = WebUser(id=222, username="o", first_name="O", role="user") +ADMIN = WebUser(id=999, username="a", first_name="A", role="admin") + + +class FakeConfigManager: + def is_admin(self, user_id): + return user_id == ADMIN.id + + +def _entry(report_id: str, filename: str, owner=None, source="r"): + entry = { + "id": report_id, + "title": f"Report {report_id}", + "filename": filename, + "created_at": "2026-08-19T00:00:00+00:00", + "source_type": "routine", + "source_name": source, + "tags": [], + "agent": "condor", + } + if owner is not None: + entry["user_id"] = owner + return entry + + +@pytest.fixture +def reports_dir(tmp_path, monkeypatch): + """One report per principal, plus a legacy entry with no owner at all.""" + directory = tmp_path / "reports" + directory.mkdir() + for name, body in ( + ("mine.html", "

mine

"), + ("theirs.html", "

theirs

"), + ("legacy.html", "

legacy

"), + ): + (directory / name).write_text(body, encoding="utf-8") + index = directory / "reports_index.json" + index.write_text( + json.dumps( + [ + _entry("mine01", "mine.html", owner=USER.id, source="mine"), + _entry("their1", "theirs.html", owner=OTHER.id, source="theirs"), + # Written before SEC-196: no user_id key at all. + _entry("legacy", "legacy.html", source="legacy"), + ] + ), + encoding="utf-8", + ) + monkeypatch.setattr(rep, "CHARTS_DIR", directory) + monkeypatch.setattr(rep, "INDEX_FILE", index) + monkeypatch.setattr(routes, "get_config_manager", lambda: FakeConfigManager()) + return directory + + +def _client(user: WebUser) -> TestClient: + app = create_app() + app.dependency_overrides[get_current_user] = lambda: user + return TestClient(app) + + +# ── Listing ── + + +def test_list_returns_only_your_own_for_a_non_admin(reports_dir): + with _client(USER) as client: + payload = client.get("/api/v1/reports").json() + assert [r["id"] for r in payload["reports"]] == ["mine01"] + assert payload["total"] == 1 + + +def test_list_returns_everything_for_an_admin(reports_dir): + with _client(ADMIN) as client: + payload = client.get("/api/v1/reports").json() + assert sorted(r["id"] for r in payload["reports"]) == [ + "legacy", + "mine01", + "their1", + ] + + +def test_grouped_listing_is_scoped_the_same_way(reports_dir): + with _client(USER) as client: + groups = client.get("/api/v1/reports/latest-by-source").json() + assert [g["source_name"] for g in groups] == ["mine"] + with _client(ADMIN) as client: + groups = client.get("/api/v1/reports/latest-by-source").json() + assert sorted(g["source_name"] for g in groups) == ["legacy", "mine", "theirs"] + + +# ── Id-addressed reads and deletes ── + + +def test_owner_reads_detail_html_and_deletes_their_own(reports_dir): + with _client(USER) as client: + assert client.get("/api/v1/reports/mine01").status_code == 200 + response = client.get("/api/v1/reports/mine01/html") + assert response.status_code == 200 + assert response.text == "

mine

" + assert client.delete("/api/v1/reports/mine01").json() == {"deleted": True} + + +def test_cross_owner_report_is_refused_everywhere(reports_dir): + """A user cannot read the metadata, body, or delete another user's report.""" + with _client(USER) as client: + assert client.get("/api/v1/reports/their1").status_code == 403 + response = client.get("/api/v1/reports/their1/html") + assert response.status_code == 403 + assert "theirs" not in response.text + assert client.delete("/api/v1/reports/their1").status_code == 403 + # And nothing was deleted. + assert rep.get_report("their1") is not None + + +def test_ownerless_legacy_entry_is_admin_only(reports_dir): + """No owner recorded -> fail closed: invisible to non-admins, not deleted.""" + with _client(USER) as client: + assert client.get("/api/v1/reports/legacy").status_code == 403 + assert client.get("/api/v1/reports/legacy/html").status_code == 403 + assert client.delete("/api/v1/reports/legacy").status_code == 403 + assert rep.get_report("legacy") is not None + with _client(ADMIN) as client: + assert client.get("/api/v1/reports/legacy").status_code == 200 + assert client.get("/api/v1/reports/legacy/html").text == "

legacy

" + + +def test_admin_reads_and_deletes_across_owners(reports_dir): + with _client(ADMIN) as client: + assert client.get("/api/v1/reports/their1/html").status_code == 200 + assert client.delete("/api/v1/reports/their1").json() == {"deleted": True} + + +def test_missing_report_is_404_not_403(reports_dir): + with _client(USER) as client: + assert client.get("/api/v1/reports/nope42").status_code == 404 + + +# ── Producer stamping ── + + +def test_save_records_the_owner_from_attribute_owner(tmp_path, monkeypatch): + monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) + monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + + async def go(): + with rep.attribute_owner(USER.id): + b = rep.ReportBuilder("Owned") + b.source("routine", "r") + b.markdown("body") + await b.save() + # Outside the block the owner resets: a bare save is ownerless. + b = rep.ReportBuilder("Bare") + b.source("routine", "r") + b.markdown("body") + await b.save() + + asyncio.run(go()) + by_title = {e["title"]: e for e in rep.list_reports()[0]} + assert by_title["Owned"]["user_id"] == USER.id + assert by_title["Bare"]["user_id"] is None + + +def test_list_reports_owner_filter_drops_foreign_and_ownerless(tmp_path, monkeypatch): + monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) + monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + + async def go(): + for owner, title in ((USER.id, "Mine"), (OTHER.id, "Theirs"), (None, "Old")): + with rep.attribute_owner(owner): + b = rep.ReportBuilder(title) + b.source("routine", "r") + b.markdown("body") + await b.save() + + asyncio.run(go()) + mine, total = rep.list_reports(owner_id=USER.id) + assert [e["title"] for e in mine] == ["Mine"] + assert total == 1 + everything, _ = rep.list_reports() + assert len(everything) == 3 + + +def test_live_report_update_preserves_the_owner(tmp_path, monkeypatch): + """An in-place update keeps the user_id stamped at first save.""" + monkeypatch.setattr(rep, "CHARTS_DIR", tmp_path) + monkeypatch.setattr(rep, "INDEX_FILE", tmp_path / "reports_index.json") + + async def go(): + live = rep.LiveReport("Live", source_name="loop") + with rep.attribute_owner(USER.id): + live.builder.markdown("tick 1") + await live.update() + live.clear() + live.builder.markdown("tick 2") + with rep.attribute_owner(USER.id): + await live.update() + return live.report_id + + report_id = asyncio.run(go()) + assert rep.get_report(report_id)["user_id"] == USER.id diff --git a/tests/test_web_report_serving.py b/tests/test_web_report_serving.py index cd7171e0e..70cd47787 100644 --- a/tests/test_web_report_serving.py +++ b/tests/test_web_report_serving.py @@ -12,12 +12,19 @@ from starlette.testclient import TestClient import condor.reports as reports +import condor.web.routes.reports as reports_routes from condor.web.app import create_app from condor.web.auth import get_current_user from condor.web.models import WebUser USER = WebUser(id=111, username="u", first_name="U", role="user") + +class _NoAdmins: + def is_admin(self, user_id): + return False + + BODY = "

Report

" # Distinctive markers so a leak is unmistakable and cannot be confused with a # 404 detail message that happens to echo the requested path back. @@ -38,6 +45,10 @@ def reports_dir(tmp_path, monkeypatch): index.write_text( json.dumps( [ + # Entries are stamped with USER's id: since SEC-196 an entry + # with no owner is admin-only, and these tests exercise the + # serving/path-containment logic, not the ownership gate + # (which has its own tests in test_reports_ownership.py). { "id": "abc123", "title": "Sample", @@ -46,6 +57,7 @@ def reports_dir(tmp_path, monkeypatch): "source_type": "routine", "source_name": "sample", "tags": [], + "user_id": 111, }, # Hostile index entries: the filename must never escape the # reports directory, and must be .html, even if the index lies. @@ -57,6 +69,7 @@ def reports_dir(tmp_path, monkeypatch): "source_type": "routine", "source_name": "escape", "tags": [], + "user_id": 111, }, { "id": "nothtml", @@ -66,6 +79,7 @@ def reports_dir(tmp_path, monkeypatch): "source_type": "routine", "source_name": "nothtml", "tags": [], + "user_id": 111, }, ] ), @@ -73,6 +87,9 @@ def reports_dir(tmp_path, monkeypatch): ) monkeypatch.setattr(reports, "CHARTS_DIR", directory) monkeypatch.setattr(reports, "INDEX_FILE", index) + # Ownership (SEC-196) is orthogonal here: USER owns every seeded entry and + # is not an admin, so these tests keep exercising serving/containment. + monkeypatch.setattr(reports_routes, "get_config_manager", lambda: _NoAdmins()) return directory From edd8aa5dfea7ade63291e37d9a08e5d47149b800 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 18:24:58 +0300 Subject: [PATCH 36/57] (feat) update telemetry and general improvements --- PRIVACY.md | 26 +++--- README.md | 8 +- condor/agents/consult.py | 46 ++++------- condor/agents/engine.py | 64 ++++----------- condor/runtime/sessions.py | 120 +++++++++------------------- condor/telemetry/__init__.py | 6 +- condor/telemetry/emitter.py | 2 - condor/telemetry/outbox.py | 21 +++-- condor/web/routes/settings.py | 7 +- tests/test_agents.py | 2 +- tests/test_chat_ws_conversations.py | 2 +- tests/test_chat_ws_prewarm.py | 2 +- tests/test_chat_ws_steering.py | 2 +- tests/test_chat_ws_wake.py | 2 +- tests/test_session_model_switch.py | 2 +- tests/test_session_server_access.py | 4 +- tests/test_session_server_switch.py | 2 +- tests/test_session_switch.py | 2 +- tests/test_telemetry.py | 15 +++- utils/config.py | 5 -- 20 files changed, 125 insertions(+), 215 deletions(-) diff --git a/PRIVACY.md b/PRIVACY.md index 3cfc426f3..fbb8d2003 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -8,9 +8,9 @@ optional exception does. **Short version:** a fresh install sends nothing. It has no telemetry consent recorded, which resolves to level `off`, at which the emitter returns before it has looked at its arguments — no buffer, no file, no directory. Nothing changes -until an admin taps "yes" on a prompt. Even then, this repository has no -collector address compiled into it: with `CONDOR_TELEMETRY_URL` unset, the send -path is inert and events can only ever reach a capped local file. +until an admin taps "yes" on a prompt. If one does, batches go to the project's +collector at `https://telemetry.hummingbot.org/v1/events`, which is fixed in the +source and cannot be pointed elsewhere; consent is the only switch. The whole mechanism is about 900 lines in [`condor/telemetry/`](condor/telemetry/). It is meant to be read, not trusted. @@ -94,13 +94,20 @@ per-install, the same person on two installs produces two unrelated hashes. ## Where it goes -Nowhere, unless you configure a destination. +Nowhere, unless you opt in. -No collector URL is compiled into this repository. Events are buffered in memory -and, when a batch cannot be delivered, appended to +If you do, batches are POSTed to `https://telemetry.hummingbot.org/v1/events`. +That address is compiled into +[`condor/telemetry/outbox.py`](condor/telemetry/outbox.py) as `COLLECTOR_URL` +and is not configurable — there is no environment variable or config key that +redirects it, so the only way to change where your data goes is to edit that +line in your own checkout. The collector itself is open source +(`condor-telemetry-server`). + +Events are buffered in memory and, when a batch cannot be delivered, appended to `condor/.runtime/telemetry/outbox.jsonl`, which is capped at 5,000 events and 7 -days — oldest dropped. With `CONDOR_TELEMETRY_URL` unset, that is the entire -life cycle: a local file, capped, that you can delete. +days — oldest dropped. At level `off` no event is ever created, so nothing is +buffered, nothing is written, and nothing is sent. You can read exactly what would be sent: @@ -121,8 +128,7 @@ Three ways to control it, in order of precedence: 1. **Environment** — `CONDOR_TELEMETRY=off` in your `.env`. This overrides everything, in both directions, and is the right answer for a headless or - containerized install. `CONDOR_TELEMETRY_URL` is what enables sending at all; - leave it unset and nothing can be transmitted. + containerized install. 2. **The dashboard API** — `PUT /api/v1/settings/telemetry?level=off` (admin only). `GET` the same path to see the current state. 3. **`config.yml`** — edit the `telemetry` section directly: diff --git a/README.md b/README.md index fac5a7711..3544e87f3 100644 --- a/README.md +++ b/README.md @@ -13,10 +13,10 @@ A Telegram bot for monitoring and trading with Hummingbot via the **Hummingbot A > Full walkthrough: [Securing Condor and Hummingbot API with Tailscale](https://hummingbot.org/blog/posts/securing-condor-and-hummingbot-api-with-tailscale/) · [Hummingbot API Tailscale guide](https://hummingbot.org/hummingbot-api/tailscale/) > **Privacy:** Condor collects nothing by default. It ships with usage telemetry -> **off** and no collector address, and it stays that way unless an admin -> explicitly opts in from the one prompt it sends on first boot. What that -> option would and would not collect — and how to verify it is off — is spelled -> out in [PRIVACY.md](PRIVACY.md). +> **off**, and it stays that way unless an admin explicitly opts in from the one +> prompt it sends on first boot. What that option would and would not collect — +> where it goes, and how to verify it is off — is spelled out in +> [PRIVACY.md](PRIVACY.md). ## Features diff --git a/condor/agents/consult.py b/condor/agents/consult.py index 5f8eab5e3..de09183ca 100644 --- a/condor/agents/consult.py +++ b/condor/agents/consult.py @@ -21,7 +21,6 @@ import os from condor.acp.pydantic_ai_client import ( - PydanticAIClient, healthcheck_local_backend, is_pydantic_ai_model, ) @@ -150,9 +149,6 @@ async def _run_agent_to_completion( fallback, ) model_key = fallback - # The fallback is a different model — re-resolve, or a custom - # endpoint's credentials would leak into an unrelated backend. - base_url, api_key = resolve_custom_endpoint(model_key, user_id=user_id) fallback_note = ( f"_(note: {agent.name}'s configured model was unavailable — " f"{backend_err} Answered with fallback `{fallback}`.)_\n\n" @@ -169,7 +165,6 @@ async def _run_agent_to_completion( from handlers.agents._shared import ( build_agent_context, build_mcp_servers_for_session, - get_project_dir, ) # A server pinned on the Agent itself wins over the ambient chat server; when @@ -192,31 +187,22 @@ async def _run_agent_to_completion( # agent auto-approves (unattended). permission_cb = permission_callback - # Build the client for the (possibly fallback) model. A pydantic-ai model gets - # the agent's tool allowlist enforced; an ACP fallback (claude-code) cannot - # enforce an allowlist, so it runs the consult unrestricted — acceptable since - # it is the trusted coordinator model and mutations are still confirmation-gated. - if is_pydantic_ai_model(model_key): - client = PydanticAIClient( - model=model_key, - mcp_servers=mcp_servers, - permission_callback=permission_cb, - allowed_tools=agent.tools or None, - base_url=base_url, - api_key=api_key, - ) - else: - from condor.acp.client import ACPClient, resolve_acp - - agent_cmd, model_env, model_pref = resolve_acp(model_key) - client = ACPClient( - command=agent_cmd, - working_dir=get_project_dir(), - mcp_servers=mcp_servers, - permission_callback=permission_cb, - extra_env=model_env or None, - model=model_pref or None, - ) + # Build the client for the (possibly fallback) model through the shared + # factory (ARCH-192). A pydantic-ai model gets the agent's tool allowlist + # enforced; an ACP fallback (claude-code) cannot enforce an allowlist, so it + # runs the consult unrestricted — acceptable since it is the trusted + # coordinator model and mutations are still confirmation-gated. The factory + # re-resolves the custom endpoint (same lenient inputs as the healthcheck + # above), so a fallback model never inherits the original's credentials. + from condor.runtime.llm_client import build_llm_client + + client = build_llm_client( + model_key, + mcp_servers=mcp_servers, + permission_callback=permission_cb, + allowed_tools=agent.tools or None, + user_id=user_id, + ) prompt = build_agent_context(agent, user_id, task, context) diff --git a/condor/agents/engine.py b/condor/agents/engine.py index acebc0e13..322c49653 100644 --- a/condor/agents/engine.py +++ b/condor/agents/engine.py @@ -24,9 +24,8 @@ ToolCallEvent, ToolCallUpdate, fold_tool_call_event, - resolve_acp, ) -from condor.acp.pydantic_ai_client import PydanticAIClient, is_pydantic_ai_model +from condor.acp.pydantic_ai_client import PydanticAIClient from condor.runtime.registry_file import LoopState from condor.runtime.timeouts import resolve_tick_timeout from condor.telemetry import taps as telemetry_taps @@ -866,10 +865,7 @@ async def _create_client( (it only feeds the auto-approve callback and cannot change between the two points), avoiding a redundant per-tick journal re-parse. """ - from handlers.agents._shared import ( - build_mcp_servers_for_session, - get_project_dir, - ) + from handlers.agents._shared import build_mcp_servers_for_session mode = self.config.get("execution_mode", "loop") @@ -888,47 +884,21 @@ async def _create_client( agent_id=self.agent_id, ) - agent_key = self._agent_key() - use_pydantic_ai = is_pydantic_ai_model(agent_key) - - if use_pydantic_ai: - import os - - from condor.preferences import resolve_custom_endpoint - - # A custom endpoint's URL/key come from the owner's saved endpoints; - # an explicit model_base_url in the run config still wins. - custom_url, api_key = resolve_custom_endpoint( - agent_key, user_id=self.user_id - ) - base_url = self.config.get("model_base_url") or custom_url or None - tool_filter_mode = ( - self.config.get("tool_filter_mode") - or os.environ.get("PYDANTIC_AI_TOOL_FILTER") - or None - ) - return PydanticAIClient( - model=agent_key, - mcp_servers=mcp_servers, - permission_callback=permission_cb, - base_url=base_url, - api_key=api_key, - tool_filter_mode=tool_filter_mode, - # Same allowlist the agent gets on consult; empty => unrestricted. - allowed_tools=self.agent.tools or None, - ) - else: - # Supports a Claude model suffix, e.g. "claude-acp:opus" — selected via - # session/set_model after handshake (the bridge ignores ANTHROPIC_MODEL). - agent_cmd, model_env, model_pref = resolve_acp(agent_key) - return ACPClient( - command=agent_cmd, - working_dir=get_project_dir(), - mcp_servers=mcp_servers, - permission_callback=permission_cb, - extra_env=model_env or None, - model=model_pref or None, - ) + # Shared factory (ARCH-192). Engine specifics: an explicit model_base_url + # in the run config still wins over the owner's saved custom endpoint, + # and the run config's tool_filter_mode beats the env fallback. Same + # allowlist the agent gets on consult; empty => unrestricted. + from condor.runtime.llm_client import build_llm_client + + return build_llm_client( + self._agent_key(), + mcp_servers=mcp_servers, + permission_callback=permission_cb, + allowed_tools=self.agent.tools or None, + user_id=self.user_id, + base_url_override=self.config.get("model_base_url") or None, + tool_filter_mode=self.config.get("tool_filter_mode"), + ) # ------------------------------------------------------------------ # Helpers diff --git a/condor/runtime/sessions.py b/condor/runtime/sessions.py index 1858f3d50..cc9b9851f 100644 --- a/condor/runtime/sessions.py +++ b/condor/runtime/sessions.py @@ -17,23 +17,15 @@ from telegram import Bot -from condor.acp import ACPClient, PermissionCallback, PromptDone, resolve_acp -from condor.acp.pydantic_ai_client import ( - PydanticAIClient, - is_pydantic_ai_model, - model_prefix, -) +from condor.acp import ACPClient, PermissionCallback, PromptDone +from condor.acp.pydantic_ai_client import PydanticAIClient from condor.agents.agent import identity_header as agent_identity_header from condor.runtime import binding, conversations from condor.runtime.confirmations import get_registry as get_confirmation_registry from condor.runtime.keys import SessionKey from condor.runtime.models import SessionInfo, SessionSpec from condor.runtime.timeouts import TIMEOUTS -from handlers.agents._shared import ( - build_initial_context, - get_project_dir, - platform_formatting, -) +from handlers.agents._shared import build_initial_context, platform_formatting log = logging.getLogger(__name__) @@ -511,80 +503,40 @@ async def _spawn_session( mcp_servers = bound.mcp_servers agent_key = bound.agent_key or spec.agent_key - # Check if agent_key requires PydanticAI client (ollama, lmstudio, openai, etc.) - use_pydantic_ai = is_pydantic_ai_model(agent_key) - - if use_pydantic_ai: - # For Pydantic AI models: auto-detect or use configured filter mode - import os - - from condor.preferences import get_agent_prefs - - # Priority: user preference > env variable > auto-detect (None) - agent_prefs = get_agent_prefs(user_data) if user_data else {} - tool_filter_mode = ( - agent_prefs.get("tool_filter_mode") - or os.environ.get("PYDANTIC_AI_TOOL_FILTER") - or None # None triggers auto-detection based on model size - ) - - base_url = ( + # One factory decides PydanticAI vs ACP for every surface (ARCH-192). + # Chat-session specifics: the CONDOR_* env pair, the bound-Agent identity + # header (system level — the opening context below is a user turn and loses + # to the host's own system prompt, FEAT-025), the saved LM Studio pref as + # the *default* base URL (a named custom endpoint still wins), and + # strict_custom_endpoint=True so a key naming an unsaved endpoint fails + # loudly with the guided RuntimeError instead of dying deep in httpx. + import os + + from condor.preferences import get_agent_prefs + from condor.runtime.llm_client import build_llm_client + + agent_prefs = get_agent_prefs(user_data) if user_data else {} + client = build_llm_client( + agent_key, + mcp_servers=mcp_servers, + permission_callback=permission_callback, + # A bound Agent's allowlist is enforced here exactly as it is on + # consult and loop, so an Agent has the same reach in every mode. + allowed_tools=bound.tools or None, + extra_env=extra_env, + system_prompt=( + agent_identity_header(bound.agent_slug, bound.label) + if bound.is_agent + else "" + ), + user_data=user_data, + user_id=spec.user_id, + default_base_url=( agent_prefs.get("base_url") or os.environ.get("LMSTUDIO_BASE_URL") or None - ) - - api_key = None - if model_prefix(agent_key) == "custom": - # Custom OpenAI-compatible provider. The agent key names one of the - # user's saved endpoints ("custom@venice:..."); those live in the - # shared preference store so Telegram and the web dashboard resolve - # them identically. CUSTOM_LLM_* env vars cover headless deploys. - # strict=True keeps the actionable RuntimeError for a missing named - # endpoint; user_id lets a session without user_data still resolve - # saved endpoints (parity with consult/engine). - from condor.preferences import resolve_custom_endpoint - - base_url, api_key = resolve_custom_endpoint( - agent_key, - user_data=user_data, - user_id=spec.user_id, - strict=True, - ) - - client = PydanticAIClient( - model=agent_key, - mcp_servers=mcp_servers, - permission_callback=permission_callback, - extra_env=extra_env, - tool_filter_mode=tool_filter_mode, # Auto-detects if None - base_url=base_url, - api_key=api_key, - # A bound Agent's allowlist is enforced here exactly as it is on - # consult and loop, so an Agent has the same reach in every mode. - allowed_tools=bound.tools or None, - ) - else: - # For ACP subprocess models: claude-code, gemini, codex. - # A Claude model can be pinned via a suffix, e.g. "claude-acp:opus" / - # "claude-acp:sonnet"; ACPClient selects it via session/set_model after - # handshake (the bridge ignores ANTHROPIC_MODEL). Bare key = agent default. - command, model_env, model_pref = resolve_acp(agent_key) - client = ACPClient( - command=command, - working_dir=get_project_dir(), - mcp_servers=mcp_servers, - permission_callback=permission_callback, - extra_env={**extra_env, **model_env}, - model=model_pref, - # Who this brain IS, at system level. The opening context below is a - # user turn: it is read once and then loses to the host's own system - # prompt, which is why a bound Agent kept answering as Condor - # (FEAT-025). This is the one channel that outranks it. - system_prompt=( - agent_identity_header(bound.agent_slug, bound.label) - if bound.is_agent - else "" - ), - ) + ), + tool_filter_mode=agent_prefs.get("tool_filter_mode"), + strict_custom_endpoint=True, + ) await client.start() diff --git a/condor/telemetry/__init__.py b/condor/telemetry/__init__.py index 577606190..b8aa35c2e 100644 --- a/condor/telemetry/__init__.py +++ b/condor/telemetry/__init__.py @@ -15,9 +15,9 @@ - **Allowlisted.** :mod:`condor.telemetry.schema` declares every event and every property. Anything undeclared is dropped by construction, which is what makes the "never collected" list in ``PRIVACY.md`` a property of the code. -- **Nothing transmits without an endpoint.** No collector URL is compiled in. - Unset ``CONDOR_TELEMETRY_URL`` means the send path is inert and events only - ever reach a capped local file. +- **Consent is the only switch.** The collector address is fixed in + :mod:`condor.telemetry.outbox`; at level ``off`` nothing is ever recorded, so + there is nothing to send it. Public surface — call sites should need nothing else:: diff --git a/condor/telemetry/emitter.py b/condor/telemetry/emitter.py index b740b5468..543dc0f82 100644 --- a/condor/telemetry/emitter.py +++ b/condor/telemetry/emitter.py @@ -159,8 +159,6 @@ async def flush(reason: str = "job") -> int: return 0 if not outbox.endpoint(): - # Nowhere to go yet (the collector is FEAT-024). Park them; the cap - # in outbox.py is what keeps this honest on a long-running install. outbox.stash(events) return 0 diff --git a/condor/telemetry/outbox.py b/condor/telemetry/outbox.py index a11b74871..52eb6d326 100644 --- a/condor/telemetry/outbox.py +++ b/condor/telemetry/outbox.py @@ -15,11 +15,10 @@ so an install that never reaches a collector accumulates a bounded file and then quietly drops the excess. That is the intended behaviour, not a bug. -**Nothing is transmitted unless an operator sets ``CONDOR_TELEMETRY_URL``.** No -collector address is compiled into this repository. With the variable unset — -which is the shipped state — :func:`post` returns ``False`` without importing a -network client, and every event's whole life is a local file that -:func:`purge` can delete. +The collector address is :data:`COLLECTOR_URL`, compiled in and not +configurable. Whether anything is sent to it is decided entirely by consent: at +level ``off`` — the default — no event is ever created, so :func:`post` is never +reached, and :func:`purge` deletes whatever a withdrawn opt-in left behind. """ from __future__ import annotations @@ -43,6 +42,7 @@ # outbox could never drain. Keep this at or below the smallest cap we target. MAX_BATCH_EVENTS = 500 POST_TIMEOUT_S = 10 +COLLECTOR_URL = "https://telemetry.hummingbot.org/v1/events" def root() -> Path: @@ -206,16 +206,13 @@ def batches(events: list[dict]) -> list[list[dict]]: return out -def endpoint() -> str | None: - """The collector URL, or None — which is the shipped default.""" - from utils.config import CONDOR_TELEMETRY_URL - - return CONDOR_TELEMETRY_URL +def endpoint() -> str: + """The collector URL.""" + return COLLECTOR_URL async def post(envelope: dict) -> bool: - """Deliver one envelope. Returns False — without touching the network — when - no endpoint is configured, which is the state this repository ships in.""" + """Deliver one envelope.""" url = endpoint() if not url: return False diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index b5b68dcfa..b00d2ce74 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -832,10 +832,9 @@ async def delete_custom_provider( async def get_telemetry_settings(user: WebUser = Depends(get_current_user)): """What this install has agreed to, and whether it could send anything. - ``endpoint_configured`` is the honest answer to "is this thing on": with no - ``CONDOR_TELEMETRY_URL`` set — the shipped state — nothing can be - transmitted no matter what the consent says, and events only accumulate in - a capped local file. + The collector address is fixed in the source, so consent is the only thing + that decides whether anything is transmitted: at level ``off`` no event is + recorded in the first place. """ from condor.telemetry import consent, emitter, outbox diff --git a/tests/test_agents.py b/tests/test_agents.py index d77231274..85e4984d5 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -544,7 +544,7 @@ def _run_create_session(monkeypatch, *, chat_id, user_id): from condor.runtime import sessions as session_module monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _FakeACPClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _FakeACPClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") # Resolved through condor.runtime.binding now, so patch it at the source. monkeypatch.setattr( diff --git a/tests/test_chat_ws_conversations.py b/tests/test_chat_ws_conversations.py index a88066758..755920534 100644 --- a/tests/test_chat_ws_conversations.py +++ b/tests/test_chat_ws_conversations.py @@ -68,7 +68,7 @@ def runtime_env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _ScriptedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _ScriptedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] diff --git a/tests/test_chat_ws_prewarm.py b/tests/test_chat_ws_prewarm.py index 1c13167f9..e234d7f0b 100644 --- a/tests/test_chat_ws_prewarm.py +++ b/tests/test_chat_ws_prewarm.py @@ -71,7 +71,7 @@ def runtime_env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _SlowClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _SlowClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr(chat_ws, "_pending_spawns", {}) monkeypatch.setattr(chat_ws, "_active_prompt_tasks", {}) diff --git a/tests/test_chat_ws_steering.py b/tests/test_chat_ws_steering.py index ec7ba80d0..aaac5a26f 100644 --- a/tests/test_chat_ws_steering.py +++ b/tests/test_chat_ws_steering.py @@ -87,7 +87,7 @@ def ws_env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _GatedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _GatedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] diff --git a/tests/test_chat_ws_wake.py b/tests/test_chat_ws_wake.py index c6470f7da..211ab4c7f 100644 --- a/tests/test_chat_ws_wake.py +++ b/tests/test_chat_ws_wake.py @@ -61,7 +61,7 @@ def ws_env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _EchoClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _EchoClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] diff --git a/tests/test_session_model_switch.py b/tests/test_session_model_switch.py index 4401369eb..e4b4a0f31 100644 --- a/tests/test_session_model_switch.py +++ b/tests/test_session_model_switch.py @@ -98,7 +98,7 @@ def env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _Client) + monkeypatch.setattr("condor.acp.client.ACPClient", _Client) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] diff --git a/tests/test_session_server_access.py b/tests/test_session_server_access.py index a8ab579fe..9655cf4c9 100644 --- a/tests/test_session_server_access.py +++ b/tests/test_session_server_access.py @@ -220,8 +220,8 @@ async def prompt(self, text): @pytest.fixture def registry(monkeypatch): monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _FakeClient) - monkeypatch.setattr(session_module, "PydanticAIClient", _FakeClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _FakeClient) + monkeypatch.setattr("condor.acp.pydantic_ai_client.PydanticAIClient", _FakeClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] diff --git a/tests/test_session_server_switch.py b/tests/test_session_server_switch.py index 29b463c24..cc4cc97ea 100644 --- a/tests/test_session_server_switch.py +++ b/tests/test_session_server_switch.py @@ -83,7 +83,7 @@ def env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _Client) + monkeypatch.setattr("condor.acp.client.ACPClient", _Client) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr(routes, "get_config_manager", lambda: FakeConfigManager()) monkeypatch.setattr( diff --git a/tests/test_session_switch.py b/tests/test_session_switch.py index afad27bfc..7ae9b75b3 100644 --- a/tests/test_session_switch.py +++ b/tests/test_session_switch.py @@ -68,7 +68,7 @@ def env(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _Client) + monkeypatch.setattr("condor.acp.client.ACPClient", _Client) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr(routes, "get_config_manager", lambda: FakeConfigManager()) monkeypatch.setattr( diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 739844a1e..0cf2ef8f9 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -25,7 +25,6 @@ def install(tmp_path, monkeypatch): monkeypatch.chdir(tmp_path) monkeypatch.setattr("utils.config.CONDOR_TELEMETRY", None, raising=False) - monkeypatch.setattr("utils.config.CONDOR_TELEMETRY_URL", None, raising=False) monkeypatch.setattr( "condor.agents.agent._DATA_ROOT", str(tmp_path / "data"), raising=False ) @@ -114,12 +113,20 @@ def test_flush_is_a_no_op_without_consent(install, monkeypatch): assert not outbox.root().exists() -def test_no_endpoint_means_events_only_ever_reach_a_local_file(install): - """The shipped state: consent granted, but nowhere to send. Nothing leaves.""" +def test_the_endpoint_is_fixed_and_not_configurable(install, monkeypatch): + """The collector address is compiled in; no env var may redirect it.""" + monkeypatch.setenv("CONDOR_TELEMETRY_URL", "http://attacker.invalid/v1/events") + + assert outbox.endpoint() == outbox.COLLECTOR_URL + assert outbox.COLLECTOR_URL == "https://telemetry.hummingbot.org/v1/events" + + +def test_no_endpoint_means_events_only_ever_reach_a_local_file(install, monkeypatch): + """With delivery impossible, events park in the capped outbox instead of leaving.""" _grant("usage") + monkeypatch.setattr(outbox, "endpoint", lambda: None) emitter.emit("command", name="portfolio", surface="telegram") - assert outbox.endpoint() is None assert asyncio.run(emitter.flush("test")) == 0 stashed = outbox.take_stashed() diff --git a/utils/config.py b/utils/config.py index c0d6e160b..0430115ec 100644 --- a/utils/config.py +++ b/utils/config.py @@ -41,8 +41,3 @@ # ping - install / heartbeat / version_change / shutdown only. # usage - the full allowlisted taxonomy in condor/telemetry/schema.py. CONDOR_TELEMETRY = os.environ.get("CONDOR_TELEMETRY", "").strip().lower() or None - -# Where a batch would be POSTed. Deliberately unset by default and NOT baked -# into the source: with no URL the send path is inert, and events can only ever -# accumulate in the local capped outbox. See PRIVACY.md. -CONDOR_TELEMETRY_URL = os.environ.get("CONDOR_TELEMETRY_URL", "").strip() or None From df22d27f8159a504759cb49785e35cb7b06cb030 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 18:25:18 +0300 Subject: [PATCH 37/57] (feat) add llm client --- condor/runtime/llm_client.py | 106 +++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 condor/runtime/llm_client.py diff --git a/condor/runtime/llm_client.py b/condor/runtime/llm_client.py new file mode 100644 index 000000000..dae5a9925 --- /dev/null +++ b/condor/runtime/llm_client.py @@ -0,0 +1,106 @@ +"""One factory for the PydanticAI-vs-ACP client split (ARCH-192). + +Every surface that talks to a model — chat sessions, consult, the trading-agent +engine — needs the same decision: a pydantic-ai key (ollama/lmstudio/openai/ +custom/...) gets an in-process :class:`PydanticAIClient`, anything else gets an +:class:`ACPClient` subprocess. That branch used to live in three drifted copies +(``runtime/sessions.py``, ``agents/consult.py``, ``agents/engine.py``); it now +lives here, and each surface passes only its own specifics: + +- chat sessions: ``extra_env`` (CONDOR_* ids), the bound-Agent ``system_prompt``, + ``strict_custom_endpoint=True`` (loud, actionable error for a missing saved + endpoint) and the LM Studio pref as ``default_base_url``. +- consult: nothing extra — it healthchecks the backend *before* calling this. +- engine: the run config's ``model_base_url`` as ``base_url_override``. + +The client classes are read as module attributes at call time so tests keep +patching the defining modules (``condor.acp.client.ACPClient`` / +``condor.acp.pydantic_ai_client.PydanticAIClient``) and intercept every surface +at once. +""" + +from __future__ import annotations + +import os +from typing import Any + +from condor.acp import client as acp_client +from condor.acp import pydantic_ai_client as pydantic_ai +from condor.preferences import resolve_custom_endpoint + + +def build_llm_client( + agent_key: str, + *, + mcp_servers: list[dict[str, Any]] | None = None, + permission_callback: acp_client.PermissionCallback | None = None, + allowed_tools: list[str] | None = None, + extra_env: dict[str, str] | None = None, + system_prompt: str = "", + user_data: dict | None = None, + user_id: int | None = None, + base_url_override: str | None = None, + default_base_url: str | None = None, + tool_filter_mode: str | None = None, + strict_custom_endpoint: bool = False, +) -> acp_client.ACPClient | pydantic_ai.PydanticAIClient: + """Build (but do not start) the right client for ``agent_key``. + + Custom-endpoint keys (``custom@:``) resolve through + :func:`condor.preferences.resolve_custom_endpoint` — pass ``user_data`` + when you have it (Telegram) or ``user_id`` when you don't (web, MCP, + background runs). ``strict_custom_endpoint=True`` keeps the chat path's + guided RuntimeError for a key naming an unsaved endpoint; the lenient + default falls back to the ``CUSTOM_LLM_*`` env vars. + + Base-URL precedence: ``base_url_override`` (an explicit config value, e.g. + the engine's ``model_base_url``) beats the resolved custom endpoint, which + beats ``default_base_url`` (a generic preference, e.g. the saved LM Studio + URL — deliberately last so it cannot shadow a named custom endpoint). + + ``tool_filter_mode`` resolves as: explicit value (a user/config pref) > + ``PYDANTIC_AI_TOOL_FILTER`` env > ``None`` (auto-detect by model size). + + ``extra_env``, ``system_prompt`` and ``allowed_tools`` are forwarded to + whichever client understands them: the ACP subprocess takes the env and the + system prompt but cannot enforce an allowlist; PydanticAI enforces the + allowlist (and takes the env for its MCP subprocesses). + """ + if pydantic_ai.is_pydantic_ai_model(agent_key): + custom_url, api_key = resolve_custom_endpoint( + agent_key, + user_data=user_data, + user_id=user_id, + strict=strict_custom_endpoint, + ) + return pydantic_ai.PydanticAIClient( + model=agent_key, + mcp_servers=mcp_servers, + permission_callback=permission_callback, + extra_env=extra_env, + base_url=base_url_override or custom_url or default_base_url or None, + api_key=api_key, + tool_filter_mode=( + tool_filter_mode or os.environ.get("PYDANTIC_AI_TOOL_FILTER") or None + ), + allowed_tools=allowed_tools, + ) + + # ACP subprocess models: claude-code, gemini, codex. A Claude model can be + # pinned via a suffix ("claude-acp:opus"); ACPClient selects it over the + # protocol (session/set_model) after handshake — the bridge ignores + # ANTHROPIC_MODEL. Lazy import: handlers/ must not load at runtime-import + # time (reload_handlers would re-execute into this graph). + from handlers.agents._shared import get_project_dir + + command, model_env, model_pref = acp_client.resolve_acp(agent_key) + merged_env = {**(extra_env or {}), **model_env} + return acp_client.ACPClient( + command=command, + working_dir=get_project_dir(), + mcp_servers=mcp_servers, + permission_callback=permission_callback, + extra_env=merged_env or None, + model=model_pref or None, + system_prompt=system_prompt, + ) From 71607808c28e99cb05ffd83d431d21230f4e9a0e Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 20:15:21 +0300 Subject: [PATCH 38/57] (fix) repoint runtime test client patches at the defining modules (ARCH-192) --- tests/runtime/test_agent_binding.py | 4 ++-- tests/runtime/test_busy_policy.py | 2 +- tests/runtime/test_conversations.py | 2 +- tests/runtime/test_session_api.py | 2 +- tests/runtime/test_session_creation_lock.py | 2 +- tests/runtime/test_session_facade.py | 2 +- tests/runtime/test_wake.py | 4 ++-- 7 files changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/runtime/test_agent_binding.py b/tests/runtime/test_agent_binding.py index 8619269d1..c2fbf47eb 100644 --- a/tests/runtime/test_agent_binding.py +++ b/tests/runtime/test_agent_binding.py @@ -64,8 +64,8 @@ def agents_root(tmp_path, monkeypatch): @pytest.fixture def registry(monkeypatch): monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _FakeClient) - monkeypatch.setattr(session_module, "PydanticAIClient", _FakeClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _FakeClient) + monkeypatch.setattr("condor.acp.pydantic_ai_client.PydanticAIClient", _FakeClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "CHAT") _FakeClient.last = None return session_module diff --git a/tests/runtime/test_busy_policy.py b/tests/runtime/test_busy_policy.py index 6633642a9..294390e0d 100644 --- a/tests/runtime/test_busy_policy.py +++ b/tests/runtime/test_busy_policy.py @@ -75,7 +75,7 @@ def registry(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _GatedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _GatedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] diff --git a/tests/runtime/test_conversations.py b/tests/runtime/test_conversations.py index 8dfbc830f..f5db3379b 100644 --- a/tests/runtime/test_conversations.py +++ b/tests/runtime/test_conversations.py @@ -674,7 +674,7 @@ async def prompt_stream(self, text): @pytest.fixture def registry(monkeypatch, conv_root): monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _ScriptedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _ScriptedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] diff --git a/tests/runtime/test_session_api.py b/tests/runtime/test_session_api.py index be91f5276..2e6ab522f 100644 --- a/tests/runtime/test_session_api.py +++ b/tests/runtime/test_session_api.py @@ -52,7 +52,7 @@ async def prompt_stream(self, text): @pytest.fixture def registry(monkeypatch): monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _ScriptedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _ScriptedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") # Lives in condor.runtime.binding now; patch it at the source so both the # bound and unbound resolution paths see the stub. diff --git a/tests/runtime/test_session_creation_lock.py b/tests/runtime/test_session_creation_lock.py index 7d71bfe65..b35b61d15 100644 --- a/tests/runtime/test_session_creation_lock.py +++ b/tests/runtime/test_session_creation_lock.py @@ -62,7 +62,7 @@ def registry(monkeypatch): monkeypatch.setattr(session_module, "_pending_creates", {}) monkeypatch.setattr(session_module, "_creation_locks", {}) monkeypatch.setattr(session_module, "_creation_lock_refs", {}) - monkeypatch.setattr(session_module, "ACPClient", _SlowClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _SlowClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] diff --git a/tests/runtime/test_session_facade.py b/tests/runtime/test_session_facade.py index 55ac3c69a..078119352 100644 --- a/tests/runtime/test_session_facade.py +++ b/tests/runtime/test_session_facade.py @@ -47,7 +47,7 @@ async def prompt(self, text): def registry(monkeypatch): """Empty registry with the subprocess + context machinery stubbed out.""" monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _FakeClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _FakeClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") # Lives in condor.runtime.binding now; patch it at the source so both the # bound and unbound resolution paths see the stub. diff --git a/tests/runtime/test_wake.py b/tests/runtime/test_wake.py index 8ace40707..53cc69378 100644 --- a/tests/runtime/test_wake.py +++ b/tests/runtime/test_wake.py @@ -101,7 +101,7 @@ def registry(tmp_path, monkeypatch): monkeypatch.setattr(conversations, "_root", lambda: tmp_path / "conversations") monkeypatch.setattr(conversations, "_live_recorders", set()) monkeypatch.setattr(session_module, "_sessions", {}) - monkeypatch.setattr(session_module, "ACPClient", _EchoClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _EchoClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] @@ -331,7 +331,7 @@ async def scenario(): @pytest.fixture def gated(registry, monkeypatch): - monkeypatch.setattr(session_module, "ACPClient", _GatedClient) + monkeypatch.setattr("condor.acp.client.ACPClient", _GatedClient) return registry From 5eeea2cdb96a0ee620fac707c8d3e58b882f083c Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 20:18:04 +0300 Subject: [PATCH 39/57] (refactor) remove dead refactor leftovers: is_hosted, should_flush, _config_path, _sum_snapshot_value, get_session_dir (READ-203) --- condor/agents/__init__.py | 3 +-- condor/agents/journal.py | 9 --------- condor/telemetry/consent.py | 7 ------- condor/telemetry/emitter.py | 9 --------- condor/web/routes/portfolio.py | 5 ----- 5 files changed, 1 insertion(+), 32 deletions(-) diff --git a/condor/agents/__init__.py b/condor/agents/__init__.py index 8ef29d838..36731d813 100644 --- a/condor/agents/__init__.py +++ b/condor/agents/__init__.py @@ -15,7 +15,7 @@ save_full_config, ) from .engine import TickEngine -from .journal import JournalManager, get_session_dir, next_session_number +from .journal import JournalManager, next_session_number from .risk import RiskEngine, RiskLimits from .strategy import Strategy, StrategyStore, split_key @@ -30,7 +30,6 @@ "save_full_config", "TickEngine", "JournalManager", - "get_session_dir", "next_session_number", "RiskEngine", "RiskLimits", diff --git a/condor/agents/journal.py b/condor/agents/journal.py index 489cd98ed..8240c0560 100644 --- a/condor/agents/journal.py +++ b/condor/agents/journal.py @@ -166,15 +166,6 @@ def resolve_agent_dirs(agent_id: str) -> tuple[Path | None, Path | None]: """ -def get_session_dir(run_key: str, session_number: int) -> Path: - """Build the path for a specific session directory. - - ``run_key`` is the ``"{agent_slug}.{strategy_slug}"`` prefix (legacy flat - slugs without a dot still resolve to ``agents/{slug}/``). - """ - return _strategy_base_dir(run_key) / "sessions" / f"session_{session_number}" - - def next_session_number(agent_dir: Path) -> int: """Determine the next session number by scanning existing session_* dirs.""" # Check new location first diff --git a/condor/telemetry/consent.py b/condor/telemetry/consent.py index d59c84167..da9935131 100644 --- a/condor/telemetry/consent.py +++ b/condor/telemetry/consent.py @@ -47,13 +47,6 @@ _cached_level: str | None = None -def _config_path() -> Path: - from config_manager import ConfigManager - - existing = ConfigManager._instance - return Path(existing.config_path if existing else "config.yml") - - def _cm(): """The ConfigManager, but only if reading it cannot create a config file.""" from config_manager import ConfigManager diff --git a/condor/telemetry/emitter.py b/condor/telemetry/emitter.py index 543dc0f82..aa84890ed 100644 --- a/condor/telemetry/emitter.py +++ b/condor/telemetry/emitter.py @@ -31,7 +31,6 @@ RING_MAX = 2000 RATE_PER_MIN = 60 -FLUSH_THRESHOLD = 200 _buffer: deque[dict] = deque(maxlen=RING_MAX) _dropped = 0 @@ -48,10 +47,6 @@ def set_hosted(value: bool) -> None: _hosted = value -def is_hosted() -> bool: - return _hosted - - def _take_token(name: str) -> bool: """Token bucket, :data:`RATE_PER_MIN` per event name per minute.""" now = time.monotonic() @@ -118,10 +113,6 @@ def dropped() -> int: return _dropped -def should_flush() -> bool: - return len(_buffer) >= FLUSH_THRESHOLD - - def drain() -> tuple[list[dict], int]: """Take the ring and the dropped count together, and reset both.""" global _dropped diff --git a/condor/web/routes/portfolio.py b/condor/web/routes/portfolio.py index 19b5829e6..82bae2f63 100644 --- a/condor/web/routes/portfolio.py +++ b/condor/web/routes/portfolio.py @@ -391,8 +391,3 @@ def _extract_connector_totals(data: object) -> dict[str, float]: elif isinstance(val, (int, float)): totals[account] = float(val) return totals - - -def _sum_snapshot_value(data: object) -> float: - """Sum USD values from a portfolio snapshot (nested account/connector/balances).""" - return sum(_extract_connector_totals(data).values()) From 0a19cadc0aecbf9a77dd5fb24332ce39a5c821f3 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 20:23:37 +0300 Subject: [PATCH 40/57] (fix) redact gateway RPC credentials from non-owner network config reads (SEC-197) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit GET /settings/gateway/networks/{id} shipped the full network config to any trader-level seat, and paid RPC providers embed the API key in the nodeURL (Helius query string, Infura/Alchemy path, or userinfo). The route now masks every secret-bearing URL part server-side for callers below the OWNER line — keeping scheme://host visible so a trader still sees which endpoint they trade against — with the same admin bypass as _require_owner, and reports a redacted flag. The frontend renders the Networks & RPC editor read-only for non-owners (OWNER_ONLY_HINT) instead of a form whose save is refused, so a masked value can never round-trip into a save; owners and admins keep the full edit flow, and POST stays owner-gated (SEC-166). --- condor/web/routes/settings.py | 54 ++++++- .../components/settings/GatewaySettings.tsx | 26 ++++ frontend/src/lib/api.ts | 2 + .../test_gateway_network_config_redaction.py | 137 ++++++++++++++++++ 4 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 tests/test_gateway_network_config_redaction.py diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index b00d2ce74..746f40040 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -2,6 +2,8 @@ import asyncio import logging +import re +from urllib.parse import urlsplit from fastapi import APIRouter, Depends, HTTPException, Query @@ -306,6 +308,46 @@ async def gateway_logs( # ── Gateway Networks (RPC) ── +_URL_VALUE = re.compile(r"^[a-z][a-z0-9+.\-]*://", re.IGNORECASE) + + +def _redact_url(value: str) -> str: + """Collapse a URL to scheme://host, hiding every part that can carry a secret. + + Paid RPC providers embed the API key either in the query string + (Helius: ``?api-key=``) or in the path (Infura ``/v3/``, + Alchemy ``/v2/``), and a URL can also carry userinfo. Rather than + pattern-match key shapes, keep only the parts that cannot hold a + credential — scheme, host, port — and mask everything else whenever any of + it is present. A bare endpoint stays readable so a trader can still tell + which node they are trading against. + """ + parts = urlsplit(value) + host = parts.hostname or "" + if parts.port: + host = f"{host}:{parts.port}" + has_secret_bearing_part = ( + parts.username + or parts.password + or parts.query + or parts.fragment + or parts.path not in ("", "/") + ) + if has_secret_bearing_part: + return f"{parts.scheme}://{host}/…" + return value + + +def _redact_network_config(value): + """Recursively mask URL-shaped strings in a network config (SEC-197).""" + if isinstance(value, dict): + return {k: _redact_network_config(v) for k, v in value.items()} + if isinstance(value, list): + return [_redact_network_config(v) for v in value] + if isinstance(value, str) and _URL_VALUE.match(value): + return _redact_url(value) + return value + @router.get("/gateway/networks") async def gateway_networks( @@ -332,10 +374,20 @@ async def gateway_network_config( user: WebUser = Depends(require_server_access_query), ): cm = get_config_manager() + # Reading gateway state stays at TRADER (see _require_owner's docstring), + # but the RPC URL in a network config embeds the owner's paid provider API + # key. Redact it server-side for anyone below the OWNER line — same admin + # bypass as _require_owner — instead of shipping the secret and hiding it + # in the UI (SEC-197). Redaction cannot corrupt a save: the POST below is + # owner-gated, and owners always receive the full values. + perm = cm.get_server_permission(user.id, server) + is_owner = perm == ServerPermission.OWNER or cm.is_admin(user.id) client = await _get_client(cm, server) try: config = await client.gateway.get_network_config(network_id) - return {"network_id": network_id, "config": config} + if not is_owner: + config = _redact_network_config(config) + return {"network_id": network_id, "config": config, "redacted": not is_owner} except Exception as e: logger.exception( "Failed to fetch gateway network config '%s' from '%s'", network_id, server diff --git a/frontend/src/components/settings/GatewaySettings.tsx b/frontend/src/components/settings/GatewaySettings.tsx index b6fb42bd6..7a42c1c70 100644 --- a/frontend/src/components/settings/GatewaySettings.tsx +++ b/frontend/src/components/settings/GatewaySettings.tsx @@ -99,6 +99,32 @@ function NetworkConfigEditor({ server, networkId }: { server: string; networkId: ([key, value]) => value !== String(config[key] ?? ""), ); + // Non-owners get server-side masked values (SEC-197) and their saves are + // refused anyway (SEC-166), so render a read-only view instead of a form + // that could only fail — or worse, round-trip a masked value into a save. + const readOnly = !isOwner || data.redacted === true; + + if (readOnly) { + return ( +
+ {editableEntries.map(([key, value]) => ( +
+ + + {String(value ?? "")} + +
+ ))} +

{OWNER_ONLY_HINT}

+
+ ); + } + const save = () => { const updates: Record = {}; for (const [key, value] of changed) { diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 70dbf985b..65fc98b65 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -920,6 +920,8 @@ export interface GatewayNetworkInfo { export interface GatewayNetworkConfig { network_id: string; config: Record; + /** True when the server masked credential-bearing URL values (non-owner read, SEC-197). */ + redacted?: boolean; } export interface GatewayWalletGroup { diff --git a/tests/test_gateway_network_config_redaction.py b/tests/test_gateway_network_config_redaction.py new file mode 100644 index 000000000..7d9938e59 --- /dev/null +++ b/tests/test_gateway_network_config_redaction.py @@ -0,0 +1,137 @@ +"""Gateway network config reads redact RPC credentials for non-owners (SEC-197). + +Reading gateway state stays at TRADER (SEC-166 pinned that line: a trader must +see what they trade against), but the network config's ``nodeURL`` embeds the +owner's paid RPC provider API key — Helius puts it in the query string, +Infura/Alchemy in the path. GET /settings/gateway/networks/{id} therefore +masks every secret-bearing part of URL-shaped values server-side for anyone +below the OWNER line, with the same admin bypass `_require_owner` grants. +The secret must never reach a non-owner's browser to be hidden in the UI. +""" + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.settings as settings_routes +from condor.web.auth import get_current_user +from condor.web.models import WebUser +from config_manager import ServerPermission + +SERVER = "alpha" +OWNER = WebUser(id=1, username="owner", first_name="O", role="user") +TRADER = WebUser(id=2, username="trader", first_name="T", role="user") +ADMIN = WebUser(id=3, username="admin", first_name="A", role="admin") + +NETWORK_ID = "solana-mainnet-beta" + +# One URL per place a provider hides the key, plus values that must survive. +CONFIG = { + "nodeURL": "https://mainnet.helius-rpc.com/?api-key=super-secret-key", + "wsURL": "wss://eth-mainnet.g.alchemy.com/v2/another-secret", + "basicAuthURL": "https://user:pass@rpc.example.com", + "bareURL": "https://api.mainnet-beta.solana.com", + "chain_id": 101, + "native_currency_symbol": "SOL", + "retries": {"docsURL": "https://mainnet.infura.io/v3/path-secret"}, +} + + +class FakeGateway: + async def get_network_config(self, network_id): + return dict(CONFIG) + + +class FakeClient: + def __init__(self): + self.gateway = FakeGateway() + + +class FakeConfigManager: + """Owner id 1, trader id 2 shared, admin id 3 with no grant on the server.""" + + def __init__(self, client): + self._client = client + + def get_server_permission(self, user_id, server_name): + if server_name != SERVER: + return None + if user_id == OWNER.id: + return ServerPermission.OWNER + if user_id == TRADER.id: + return ServerPermission.TRADER + return None + + def has_server_access(self, user_id, server_name, min_permission=None): + if self.is_admin(user_id): + return server_name == SERVER + return self.get_server_permission(user_id, server_name) is not None + + def is_admin(self, user_id): + return user_id == ADMIN.id + + async def get_client(self, server_name): + return self._client + + +@pytest.fixture +def env(monkeypatch): + client = FakeClient() + cm = FakeConfigManager(client) + monkeypatch.setattr(settings_routes, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + app = FastAPI() + app.include_router(settings_routes.router) + return app + + +def as_user(app, user): + app.dependency_overrides[get_current_user] = lambda: user + return TestClient(app) + + +def get_config(app, user): + resp = as_user(app, user).get( + f"/settings/gateway/networks/{NETWORK_ID}", params={"server": SERVER} + ) + assert resp.status_code == 200 + return resp.json() + + +def test_trader_never_receives_credential_bearing_url_parts(env): + """Query keys, path keys and userinfo are all masked before the wire.""" + body = get_config(env, TRADER) + config = body["config"] + + assert body["redacted"] is True + # Helius-style query-string key. + assert config["nodeURL"] == "https://mainnet.helius-rpc.com/…" + assert "super-secret-key" not in str(config) + # Alchemy/Infura-style path key — including one nested a level down. + assert config["wsURL"] == "wss://eth-mainnet.g.alchemy.com/…" + assert "another-secret" not in str(config) + assert config["retries"]["docsURL"] == "https://mainnet.infura.io/…" + assert "path-secret" not in str(config) + # Userinfo credentials. + assert config["basicAuthURL"] == "https://rpc.example.com/…" + assert "pass" not in config["basicAuthURL"] + + +def test_trader_still_sees_the_endpoint_host_and_plain_values(env): + """Redaction hides the secret, not the network: hosts and scalars survive.""" + config = get_config(env, TRADER)["config"] + + # A bare endpoint carries no secret and stays fully readable. + assert config["bareURL"] == "https://api.mainnet-beta.solana.com" + assert config["chain_id"] == 101 + assert config["native_currency_symbol"] == "SOL" + + +@pytest.mark.parametrize("user", [OWNER, ADMIN], ids=["owner", "admin"]) +def test_owner_line_reads_full_values(env, user): + """Owners edit these values, so they receive them verbatim; admins keep + the bypass `_require_owner` grants everywhere else in this module.""" + body = get_config(env, user) + + assert body["redacted"] is False + assert body["config"] == CONFIG From 940bb8922f6a4157731a5e4a2333ce11749defc5 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 20:34:24 +0300 Subject: [PATCH 41/57] (refactor) decouple pool_data from the Telegram handler cache fetch_ohlcv and fetch_liquidity_bins no longer take user_data/chat_id/context: a pool's candles and bins are global answers, so both now cache process-wide in pool_data's own TTL caches, keyed on the upstream arguments. fetch_ohlcv gains use_cache=False (the live poll's freshness bypass, previously spelled user_data=None); fetch_liquidity_bins requires an already-resolved client, so client resolution moves to the callers like every fetchers/ module. The '_cache'-namespace stay-in-step contract with handlers/dex/_shared is gone, and dex_candles drops its module-dict-as-user_data workaround. (ARCH-194) --- condor/dex_candles.py | 11 +- condor/pool_data.py | 136 +++++++++--------------- handlers/dex/geckoterminal.py | 9 +- handlers/dex/pools.py | 9 +- tests/test_dex_liquidity_bins_route.py | 55 ++++------ tests/test_ohlcv_upstream_coalescing.py | 49 ++++++--- 6 files changed, 110 insertions(+), 159 deletions(-) diff --git a/condor/dex_candles.py b/condor/dex_candles.py index 86bed8c57..4113df163 100644 --- a/condor/dex_candles.py +++ b/condor/dex_candles.py @@ -32,10 +32,6 @@ # apart from a plain ticker like "BTC". ADDRESS_RE = re.compile(r"^(0x[0-9a-fA-F]{40}|[1-9A-HJ-NP-Za-km-z]{32,44})$") -# GeckoTerminal answers are shared across users, so one process-wide dict backs the -# OHLCV cache for every dashboard viewer. -_ohlcv_cache: dict = {} - def uses_gecko_candles(connector: str) -> bool: """Whether this connector's candles come from GeckoTerminal, not the API.""" @@ -72,9 +68,10 @@ async def _pool_ohlcv( # Prices in the quote token, matching the scale of the entry and range # prices the executor overlays draw on the same chart. currency="token", - # A live poll wants the newest candle, and the shared cache holds a - # pool for five minutes — long enough to freeze a streaming chart. - user_data=_ohlcv_cache if use_cache else None, + # A live poll wants the newest candle, and pool_data's shared cache + # holds a pool for five minutes — long enough to freeze a streaming + # chart — so use_cache=False bypasses the cached copy. + use_cache=use_cache, limit=limit, before_timestamp=before_timestamp, token=token, diff --git a/condor/pool_data.py b/condor/pool_data.py index cb2cf3796..114a4e450 100644 --- a/condor/pool_data.py +++ b/condor/pool_data.py @@ -26,17 +26,9 @@ from glom import glom from condor import orca_api -from condor.cache import evict_expired, get_cached, set_cached -from config_manager import get_client logger = logging.getLogger(__name__) -# The conversation cache these functions write into is shared with the Telegram DEX -# handlers, which read and invalidate it by group under the "_cache" namespace (see -# handlers/dex/_shared). Entries written under any other namespace would survive an -# invalidation the handler thinks it performed, so this must stay in step with it. -_CACHE_NS = "_cache" - # CLMM venues an ``lp_executor`` can open a position on: (brand, the qualifiers its # GeckoTerminal dex id must carry, the gecko networks it exists on). Deliberately # separate from LIQUIDITY_SUPPORTED_DEXES below, which answers a different question @@ -453,6 +445,8 @@ def reset_gecko_throttle() -> None: _token_symbol_cache, _token_pool_cache, _pair_pool_cache, + _ohlcv_cache, + _pool_bins_cache, ): cache.clear() @@ -529,9 +523,8 @@ def _clear(finished: "asyncio.Task", _key: Tuple = key) -> None: # ── Small TTL caches for token lookups ── -# These answers are process-wide (not per-user) and tiny, so they live here rather -# than in a caller's user_data. Capped so an unbounded stream of distinct mints -# cannot grow them without limit. +# These answers are process-wide (not per-user) and tiny. Capped so an unbounded +# stream of distinct mints cannot grow them without limit. _TOKEN_CACHE_MAX = 512 _token_symbol_cache: Dict[Tuple[str, str], Tuple[float, str]] = {} # Both hold normalized pool dicts, and an empty dict for "asked, there is no such @@ -948,22 +941,22 @@ async def fetch_ohlcv( network: str, timeframe: str = "1h", currency: str = "usd", - user_data: dict = None, limit: int = 100, before_timestamp: Optional[int] = None, token: str = "base", + use_cache: bool = True, ) -> Tuple[Optional[List], Optional[str]]: """Fetch OHLCV data for any pool via GeckoTerminal - Concurrent callers asking for the same upstream window share one request, no - matter what key each of them caches the answer under. + A pool's candles are global, not per-user, so the answer is cached once for + the process; concurrent callers asking for the same upstream window share one + request. Args: pool_address: Pool contract address network: Network identifier (will be converted to GeckoTerminal format) timeframe: OHLCV timeframe (see GECKO_TIMEFRAMES) currency: Price currency - "usd" or "token" (quote token) - user_data: Optional user_data dict for caching limit: Number of candles to fetch (capped at GECKO_OHLCV_MAX) before_timestamp: Unix seconds; candles walk back from here. None asks for the latest candles, which is also what keeps a live chart's cache key @@ -971,6 +964,10 @@ async def fetch_ohlcv( token: Which side of the pool to price, "base" or "quote". "quote" flips the series, for a venue pair quoted the other way round from the pool (``XRP-RLUSD`` against a ``RLUSD / XRP`` pool). + use_cache: False skips the cache *read* — for a live poll that must see + the newest candle rather than one up to ``OHLCV_CACHE_TTL`` old. The + fresh answer is still written back, so cached readers only get newer + data out of it. Returns: Tuple of (ohlcv_list, error_message) @@ -982,23 +979,24 @@ async def fetch_ohlcv( timeframe = normalize_timeframe(timeframe) limit = max(1, min(int(limit), GECKO_OHLCV_MAX)) - # Check cache. limit/before_timestamp belong to the key: the same pool - # charted over a historical window and over the live one are different - # answers. - if user_data is not None: - cache_key = ( - f"ohlcv_{gecko_network}_{pool_address}_{timeframe}_{currency}" - f"_{token}_{limit}_{before_timestamp or 0}" - ) - cached = get_cached( - user_data, cache_key, ttl=OHLCV_CACHE_TTL, namespace=_CACHE_NS - ) + # limit/before_timestamp belong to the key: the same pool charted over a + # historical window and over the live one are different answers. The same + # tuple keys the cache and the single-flight, so "one upstream window" is + # one entry everywhere. + cache_key = ( + "ohlcv", + gecko_network, + pool_address, + timeframe, + currency, + token, + limit, + before_timestamp or 0, + ) + if use_cache: + cached = _ttl_get(_ohlcv_cache, cache_key, OHLCV_CACHE_TTL) if cached is not None: return cached, None - # Sweep on miss, as cached_call does. Historical windows mint a fresh - # key per executor, so a long-lived caller would otherwise accumulate - # one entry per chart forever. - evict_expired(user_data, namespace=_CACHE_NS) async def _fetch_upstream() -> Optional[List]: # Pass all parameters explicitly: @@ -1054,34 +1052,17 @@ async def _fetch_upstream() -> Optional[List]: return rows - # Coalesce on the UPSTREAM window — pool, timeframe, currency, priced side - # and window — never on the caller's cache key. The TTL caches above are - # per-caller and only help once an answer has landed, so two callers whose - # own keys differ (a chart keyed by trading pair and one keyed by pool, the - # dashboard's shared cache and a chat's user_data) still resolve to the very - # same GeckoTerminal window and would each spend a request for it. Keyed - # here, they collapse onto one. A fetch that raises reaches every waiter and - # is cached by nobody. The rows are shared between callers: read-only. - ohlcv_list = await _single_flight( - ( - "ohlcv", - gecko_network, - pool_address, - timeframe, - currency, - token, - limit, - before_timestamp or 0, - ), - _fetch_upstream, - ) + # Coalesce on the upstream window: a TTL cache only helps once an answer + # has landed, so concurrent callers of the same window (a cache-bypassing + # live poll racing a chart render, say) would each spend a request without + # this. A fetch that raises reaches every waiter and is cached by nobody. + # The rows are shared between callers: read-only. + ohlcv_list = await _single_flight(cache_key, _fetch_upstream) if not ohlcv_list: return None, "No OHLCV data available" - # Cache result - if user_data is not None: - set_cached(user_data, cache_key, ohlcv_list, namespace=_CACHE_NS) + _ttl_put(_ohlcv_cache, cache_key, ohlcv_list, OHLCV_CACHE_TTL) return ohlcv_list, None @@ -1090,11 +1071,13 @@ async def _fetch_upstream() -> Optional[List]: return None, f"Failed to fetch OHLCV: {str(e)}" -# Bins for one pool, keyed ``(connector, pool_address)``. Used when no caller -# supplies a ``user_data`` to cache in — i.e. by the web dashboard, where every -# viewer of a pool would otherwise open its own Gateway call at the same cadence. -# Same one-minute TTL as the per-chat cache: bins move with every swap through -# the active bin, so a longer one would draw liquidity that has already left. +# OHLCV per upstream window and bins per ``(connector, pool_address)``. Both are +# global answers — a pool's candles and liquidity do not depend on who is asking — +# so they are cached once for the process and every viewer (web dashboard, any +# Telegram chat) shares one upstream call per window. Bins keep a one-minute TTL: +# they move with every swap through the active bin, so a longer one would draw +# liquidity that has already left. +_ohlcv_cache: Dict[Tuple, Tuple[float, List]] = {} _pool_bins_cache: Dict[Tuple[str, str], Tuple[float, Dict[str, Any]]] = {} @@ -1102,9 +1085,6 @@ async def fetch_liquidity_bins( pool_address: str, connector: str = "meteora", network: str = "solana-mainnet-beta", - user_data: dict = None, - chat_id: int = None, - context=None, client=None, ) -> Tuple[Optional[List], Optional[Dict], Optional[str]]: """Fetch liquidity bin data for CLMM pools via gateway @@ -1113,13 +1093,10 @@ async def fetch_liquidity_bins( pool_address: Pool contract address connector: DEX connector (meteora, raydium, orca) network: Network identifier - user_data: Optional user_data dict for caching. When omitted the result - is cached process-wide instead, so callers with no conversation of - their own (the web dashboard) still share one Gateway call per pool. - chat_id: Chat ID for per-chat server selection - client: An already-resolved API client. The web side holds a server - *name* and resolves its client through the config manager, which the - ``get_client(chat_id)`` path below cannot express. + client: An already-resolved API client, required to reach Gateway. + Resolving one is the caller's job — the web side holds a server + *name*, a Telegram handler a chat id, and each has its own accessor + for turning that into a client. Returns: Tuple of (bins_list, pool_info, error_message) @@ -1131,21 +1108,10 @@ async def fetch_liquidity_bins( if not can_fetch_liquidity(connector): return None, None, f"Liquidity data not available for {connector}" - # Check cache - cache_key = f"pool_bins_{connector}_{pool_address}" - if user_data is not None: - cached = get_cached( - user_data, cache_key, ttl=BINS_CACHE_TTL, namespace=_CACHE_NS - ) - else: - cached = _ttl_get( - _pool_bins_cache, (connector, pool_address), BINS_CACHE_TTL - ) + cached = _ttl_get(_pool_bins_cache, (connector, pool_address), BINS_CACHE_TTL) if cached is not None: return cached.get("bins"), cached, None - if client is None: - client = await get_client(chat_id, context=context) if not client: return None, None, "Gateway client not available" @@ -1205,13 +1171,7 @@ async def fetch_liquidity_bins( bins = pool_info.get("bins", []) - # Cache result - if user_data is not None: - set_cached(user_data, cache_key, pool_info, namespace=_CACHE_NS) - else: - _ttl_put( - _pool_bins_cache, (connector, pool_address), pool_info, BINS_CACHE_TTL - ) + _ttl_put(_pool_bins_cache, (connector, pool_address), pool_info, BINS_CACHE_TTL) return bins, pool_info, None diff --git a/handlers/dex/geckoterminal.py b/handlers/dex/geckoterminal.py index 7255edeab..7e0a389b3 100644 --- a/handlers/dex/geckoterminal.py +++ b/handlers/dex/geckoterminal.py @@ -25,6 +25,7 @@ gecko_request, get_connector_for_dex, ) +from config_manager import get_client from utils.telegram_formatters import escape_markdown_v2 from ._shared import cached_call @@ -2303,9 +2304,7 @@ async def show_gecko_liquidity( bins, pool_info, error = await fetch_liquidity_bins( pool_address=address, connector=connector, - user_data=context.user_data, - chat_id=chat_id, - context=context, + client=await get_client(chat_id, context=context), ) if error or not bins: @@ -2454,9 +2453,7 @@ async def show_gecko_combined( bins, pool_info, _ = await fetch_liquidity_bins( pool_address=address, connector=connector, - user_data=context.user_data, - chat_id=chat_id, - context=context, + client=await get_client(chat_id, context=context), ) if not ohlcv_data and not bins: diff --git a/handlers/dex/pools.py b/handlers/dex/pools.py index b66e94001..0c7b7c336 100644 --- a/handlers/dex/pools.py +++ b/handlers/dex/pools.py @@ -1139,7 +1139,6 @@ async def fetch_ohlcv_task(): network=gecko_network, timeframe=timeframe, currency="token", - user_data=context.user_data, ) if error: logger.warning(f"Failed to fetch OHLCV: {error}") @@ -2026,7 +2025,6 @@ async def handle_pool_ohlcv( network=network, timeframe=timeframe, currency="token", - user_data=context.user_data, ) if error or not ohlcv_data: @@ -2238,7 +2236,6 @@ async def handle_pool_combined_chart( network=network, timeframe=timeframe, currency="token", - user_data=context.user_data, ) # Get bins from cached pool_info or fetch @@ -2248,9 +2245,7 @@ async def handle_pool_combined_chart( bins, _, _ = await fetch_liquidity_bins( pool_address=pool_address, connector=connector, - user_data=context.user_data, - chat_id=chat_id, - context=context, + client=await get_client(chat_id, context=context), ) if not ohlcv_data and not bins: @@ -2930,7 +2925,6 @@ async def fetch_ohlcv_task(): network=gecko_network, timeframe=timeframe, currency="token", - user_data=context.user_data, ) return data or [] except Exception as e: @@ -4365,7 +4359,6 @@ async def show_add_position_menu( network=gecko_network, timeframe=timeframe, currency="token", - user_data=context.user_data, ) if ohlcv_error: logger.warning(f"OHLCV fetch error: {ohlcv_error}") diff --git a/tests/test_dex_liquidity_bins_route.py b/tests/test_dex_liquidity_bins_route.py index 31873f9ef..88e3510b5 100644 --- a/tests/test_dex_liquidity_bins_route.py +++ b/tests/test_dex_liquidity_bins_route.py @@ -1,9 +1,9 @@ """Liquidity bins for the depth column beside a pool's chart (FEAT-045). -Two things are pinned here. First, that ``fetch_liquidity_bins`` can be handed a -client — the web side holds a server *name*, not a chat id, and the -``get_pool_info`` → ``get_pools`` fallback for pools outside Gateway's DLMM list -must keep working through that door. Second, that the route degrades: an +Two things are pinned here. First, that ``fetch_liquidity_bins`` works through an +injected client — resolving one is the caller's job, and the ``get_pool_info`` → +``get_pools`` fallback for pools outside Gateway's DLMM list must keep working +through that door. Second, that the route degrades: an unsupported venue, an unreachable server and an upstream raise are all states the workspace renders, never a 502 the user cannot act on. """ @@ -117,14 +117,10 @@ def bins_url(connector="meteora", network="solana-mainnet-beta", pool=POOL): return f"/servers/srv/dex/pools/{pool}/bins?network={network}&connector={connector}" -# ── fetch_liquidity_bins takes a client ── +# ── fetch_liquidity_bins requires a client ── -def test_an_injected_client_replaces_the_telegram_accessor(monkeypatch): - def boom(*a, **kw): - raise AssertionError("get_client must not be reached when a client is given") - - monkeypatch.setattr(pool_data, "get_client", boom) +def test_an_injected_client_reaches_gateway(): clmm = FakeClmm(pool_info()) bins, info, error = run( pool_data.fetch_liquidity_bins(POOL, "meteora", client=FakeClient(clmm)) @@ -135,34 +131,25 @@ def boom(*a, **kw): assert clmm.calls[0][0] == "info" -def test_the_telegram_path_still_resolves_its_own_client(monkeypatch): - clmm = FakeClmm(pool_info()) - seen = {} +def test_a_missing_client_degrades_without_a_raise(): + bins, info, error = run(pool_data.fetch_liquidity_bins(POOL, "meteora")) + assert (bins, info) == (None, None) + assert "client" in error.lower() - async def fake_get_client(chat_id, context=None): - seen["chat_id"] = chat_id - return FakeClient(clmm) - monkeypatch.setattr(pool_data, "get_client", fake_get_client) - user_data = {} - bins, _info, error = run( - pool_data.fetch_liquidity_bins(POOL, "meteora", user_data=user_data, chat_id=7) +def test_two_viewers_share_one_gateway_call(): + # Distinct clients — a chat's and a dashboard viewer's — because the cache is + # keyed by pool, never by who is asking: the second read costs no Gateway call. + first_clmm, second_clmm = FakeClmm(pool_info()), FakeClmm(pool_info()) + first = run( + pool_data.fetch_liquidity_bins(POOL, "meteora", client=FakeClient(first_clmm)) + ) + second = run( + pool_data.fetch_liquidity_bins(POOL, "meteora", client=FakeClient(second_clmm)) ) - assert error is None and len(bins) == 3 - assert seen["chat_id"] == 7 - # Still cached in the caller's conversation, not in the process-wide dict. - assert pool_data._pool_bins_cache == {} - run(pool_data.fetch_liquidity_bins(POOL, "meteora", user_data=user_data, chat_id=7)) - assert len(clmm.calls) == 1 - - -def test_two_clientless_viewers_share_one_gateway_call(): - clmm = FakeClmm(pool_info()) - client = FakeClient(clmm) - first = run(pool_data.fetch_liquidity_bins(POOL, "meteora", client=client)) - second = run(pool_data.fetch_liquidity_bins(POOL, "meteora", client=client)) assert first[0] == second[0] - assert len(clmm.calls) == 1 + assert len(first_clmm.calls) == 1 + assert second_clmm.calls == [] def test_the_get_pools_fallback_survives_the_client_parameter(): diff --git a/tests/test_ohlcv_upstream_coalescing.py b/tests/test_ohlcv_upstream_coalescing.py index cf53274a7..203cb7113 100644 --- a/tests/test_ohlcv_upstream_coalescing.py +++ b/tests/test_ohlcv_upstream_coalescing.py @@ -12,7 +12,7 @@ import pytest import condor.web.routes.market as market -from condor import dex_candles, pool_data +from condor import pool_data from condor.web.models import WebUser POOL = "So11111111111111111111111111111111111111112" @@ -24,7 +24,7 @@ def _clean_state(): def _reset(): market._candle_cache.clear() market._candle_inflight.clear() - dex_candles._ohlcv_cache.clear() + # Also clears pool_data's process-wide OHLCV cache. pool_data.reset_gecko_throttle() _reset() @@ -105,23 +105,23 @@ async def scenario(): assert results[0], "both callers get the candles, not an empty chart" -def test_distinct_caller_caches_still_share_one_fetch(monkeypatch): - """Dedup keys on the upstream request, never on where the caller caches it.""" +def test_cache_bypassing_and_cached_callers_share_one_fetch(monkeypatch): + """Dedup keys on the upstream request, whether or not a caller reads the cache.""" async def scenario(): gate = asyncio.get_running_loop().create_future() gecko = _CountingGecko(gate=gate) monkeypatch.setattr(pool_data, "gecko_call", gecko) - # Three callers: two with their own private caches, one with none at all. - caches = [{}, {}, None] + # Three concurrent callers, one of them a live poll bypassing the cache: + # in flight, they still converge on one upstream request. tasks = [ asyncio.ensure_future( pool_data.fetch_ohlcv( - POOL, "meteora", timeframe="1m", user_data=cache, limit=50 + POOL, "meteora", timeframe="1m", limit=50, use_cache=use_cache ) ) - for cache in caches + for use_cache in (True, True, False) ] for _ in range(6): await asyncio.sleep(0) @@ -134,6 +134,28 @@ async def scenario(): assert all(err is None and rows == ROWS for rows, err in results) +def test_a_live_poll_skips_the_cache_read_but_refreshes_it(monkeypatch): + """use_cache=False must reach upstream even when a cached copy exists.""" + + async def scenario(): + gecko = _CountingGecko() + monkeypatch.setattr(pool_data, "gecko_call", gecko) + + await pool_data.fetch_ohlcv(POOL, "meteora", timeframe="1m", limit=50) + cached_calls = gecko.calls + await pool_data.fetch_ohlcv(POOL, "meteora", timeframe="1m", limit=50) + after_cached_read = gecko.calls + await pool_data.fetch_ohlcv( + POOL, "meteora", timeframe="1m", limit=50, use_cache=False + ) + return cached_calls, after_cached_read, gecko.calls + + first, second, third = asyncio.run(scenario()) + assert first == 1, "the first read goes upstream" + assert second == 1, "a cached reader spends nothing" + assert third == 2, "the live poll bypasses the cached copy" + + def test_a_different_window_is_not_coalesced(monkeypatch): """Same pool, different candle count: two genuinely different answers.""" @@ -165,12 +187,9 @@ async def scenario(): failing = _CountingGecko(gate=gate, fail=True) monkeypatch.setattr(pool_data, "gecko_call", failing) - cache = {} tasks = [ asyncio.ensure_future( - pool_data.fetch_ohlcv( - POOL, "meteora", timeframe="1m", user_data=cache, limit=50 - ) + pool_data.fetch_ohlcv(POOL, "meteora", timeframe="1m", limit=50) ) for _ in range(3) ] @@ -179,12 +198,10 @@ async def scenario(): gate.set_result(None) failed = await asyncio.gather(*tasks) - # Nothing was written to the caller's cache, so the retry goes upstream. + # Nothing was written to the shared cache, so the retry goes upstream. healthy = _CountingGecko() monkeypatch.setattr(pool_data, "gecko_call", healthy) - retried = await pool_data.fetch_ohlcv( - POOL, "meteora", timeframe="1m", user_data=cache, limit=50 - ) + retried = await pool_data.fetch_ohlcv(POOL, "meteora", timeframe="1m", limit=50) return failing.calls, failed, healthy.calls, retried fail_calls, failed, retry_calls, retried = asyncio.run(scenario()) From 3244c24282daab2c38478a782202286da130a1e7 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 20:51:57 +0300 Subject: [PATCH 42/57] (refactor) move agent runtime foundations from handlers/ into condor/ (ARCH-190) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit condor/ core no longer imports from handlers/: the platform-neutral pieces of handlers/agents/_shared.py moved down — MCP toolset builders to condor/runtime/toolsets.py, danger classification + confirmation summary to condor/runtime/danger.py, context builders to condor/runtime/context.py, the model catalog to condor/llm/options.py, TelegramChannel to condor/runtime/channels.py — and readiness/openrouter_models/custom_models moved wholesale to condor/llm/. clean_config_for_save moved to condor/controller_configs.py. handlers modules re-export (or sys.modules- alias) the moved names so Telegram callers are unchanged; the lazy function-local imports across condor/runtime, condor/agents and condor/web are now normal top-level imports, and the trade-gate danger lists no longer live in a hot-reloaded handlers module. --- condor/agents/consult.py | 13 +- condor/agents/engine.py | 5 +- condor/agents/risk.py | 17 +- condor/controller_configs.py | 51 ++ condor/fetchers/__init__.py | 2 +- condor/llm/__init__.py | 14 + condor/llm/custom_models.py | 344 +++++++++ condor/llm/openrouter_models.py | 119 ++++ condor/llm/options.py | 83 +++ condor/llm/readiness.py | 298 ++++++++ condor/runtime/binding.py | 9 +- condor/runtime/channels.py | 48 ++ condor/runtime/confirmations.py | 11 +- condor/runtime/context.py | 289 ++++++++ condor/runtime/danger.py | 198 ++++++ condor/runtime/llm_client.py | 6 +- condor/runtime/sessions.py | 2 +- condor/runtime/toolsets.py | 240 +++++++ condor/setup_llm.py | 10 +- condor/web/routes/agents.py | 2 +- condor/web/routes/bots.py | 2 +- condor/web/routes/chat_ws.py | 4 +- condor/web/routes/sessions.py | 2 +- condor/web/routes/settings.py | 12 +- handlers/agents/_shared.py | 739 ++------------------ handlers/agents/confirmation.py | 115 +-- handlers/agents/custom_models.py | 345 +-------- handlers/agents/openrouter_models.py | 120 +--- handlers/agents/readiness.py | 297 +------- handlers/bots/_shared.py | 46 +- tests/runtime/test_agent_binding.py | 18 +- tests/runtime/test_busy_policy.py | 2 +- tests/runtime/test_confirmations.py | 2 +- tests/runtime/test_conversations.py | 2 +- tests/runtime/test_session_api.py | 2 +- tests/runtime/test_session_creation_lock.py | 2 +- tests/runtime/test_session_facade.py | 2 +- tests/runtime/test_wake.py | 2 +- tests/test_agents.py | 2 +- tests/test_chat_ws_conversations.py | 2 +- tests/test_chat_ws_prewarm.py | 2 +- tests/test_chat_ws_steering.py | 2 +- tests/test_chat_ws_wake.py | 2 +- tests/test_delegate_worker.py | 4 +- tests/test_session_model_switch.py | 4 +- tests/test_session_server_access.py | 2 +- tests/test_session_server_switch.py | 2 +- tests/test_session_switch.py | 2 +- 48 files changed, 1829 insertions(+), 1670 deletions(-) create mode 100644 condor/controller_configs.py create mode 100644 condor/llm/__init__.py create mode 100644 condor/llm/custom_models.py create mode 100644 condor/llm/openrouter_models.py create mode 100644 condor/llm/options.py create mode 100644 condor/llm/readiness.py create mode 100644 condor/runtime/channels.py create mode 100644 condor/runtime/context.py create mode 100644 condor/runtime/danger.py create mode 100644 condor/runtime/toolsets.py diff --git a/condor/agents/consult.py b/condor/agents/consult.py index de09183ca..2a7b0620b 100644 --- a/condor/agents/consult.py +++ b/condor/agents/consult.py @@ -26,6 +26,9 @@ ) from condor.agents.agent import AgentStore from condor.preferences import resolve_custom_endpoint +from condor.runtime import context as runtime_context +from condor.runtime import toolsets +from condor.runtime.channels import TelegramChannel log = logging.getLogger(__name__) @@ -42,7 +45,6 @@ def _build_consult_permission_cb(slug: str, user_id: int, chat_id: int): try: from condor.routine_store import get_routine_store from condor.runtime.confirmations import build_permission_callback - from handlers.agents.confirmation import TelegramChannel bot = get_routine_store().get_bot() if bot is not None: @@ -162,11 +164,6 @@ async def _run_agent_to_completion( # Build the Agent's MCP toolset in the main process (ConfigManager is here). # agent_slug scopes the condor MCP tools' memory/skills to this Agent (its brain). - from handlers.agents._shared import ( - build_agent_context, - build_mcp_servers_for_session, - ) - # A server pinned on the Agent itself wins over the ambient chat server; when # the agent isn't pinned, fall back to the caller's (chat's) resolved server. # Passing server_name=None lets the builder resolve the chat's server. @@ -174,7 +171,7 @@ async def _run_agent_to_completion( # agent_slug the condor MCP tools would target the CHAT's stores. effective_server = agent.server_name or server_name - mcp_servers = build_mcp_servers_for_session( + mcp_servers = toolsets.build_mcp_servers_for_session( user_id, chat_id, server_name=effective_server if agent.server_required else None, @@ -204,7 +201,7 @@ async def _run_agent_to_completion( user_id=user_id, ) - prompt = build_agent_context(agent, user_id, task, context) + prompt = runtime_context.build_agent_context(agent, user_id, task, context) await client.start() try: diff --git a/condor/agents/engine.py b/condor/agents/engine.py index 322c49653..94a75d0fc 100644 --- a/condor/agents/engine.py +++ b/condor/agents/engine.py @@ -26,6 +26,7 @@ fold_tool_call_event, ) from condor.acp.pydantic_ai_client import PydanticAIClient +from condor.runtime import toolsets from condor.runtime.registry_file import LoopState from condor.runtime.timeouts import resolve_tick_timeout from condor.telemetry import taps as telemetry_taps @@ -865,12 +866,10 @@ async def _create_client( (it only feeds the auto-approve callback and cannot change between the two points), avoiding a redundant per-tick journal re-parse. """ - from handlers.agents._shared import build_mcp_servers_for_session - mode = self.config.get("execution_mode", "loop") # A configured server pins the toolset; None falls back to the chat's. - mcp_servers = build_mcp_servers_for_session( + mcp_servers = toolsets.build_mcp_servers_for_session( self.user_id, self.chat_id, server_name=self.config.get("server_name"), diff --git a/condor/agents/risk.py b/condor/agents/risk.py index 39018f900..07124931b 100644 --- a/condor/agents/risk.py +++ b/condor/agents/risk.py @@ -11,6 +11,13 @@ from dataclasses import dataclass, field from typing import TYPE_CHECKING, Any +from condor.runtime.danger import ( + DANGEROUS_BOT_ACTIONS, + is_dangerous_tool_call, + tool_call_input, + tool_call_name, +) + if TYPE_CHECKING: from .ownership import BotLedger @@ -136,8 +143,6 @@ def check_executor_action( Returns (allowed, reason). """ - from handlers.agents._shared import tool_call_input - input_data = tool_call_input(tool_call) if input_data is None: return False, "Tool arguments could not be read" @@ -186,8 +191,6 @@ def check_bot_action(self, tool_call: dict) -> tuple[bool, str]: Returns (allowed, reason). """ - from handlers.agents._shared import tool_call_input - input_data = tool_call_input(tool_call) if input_data is None: return False, "Tool arguments could not be read" @@ -240,12 +243,6 @@ def auto_approve_with_risk_check( mistyped one open a live position no session can ever claim. Empty (consults, chat, tests) keeps the presence-only check. """ - from handlers.agents._shared import ( - DANGEROUS_BOT_ACTIONS, - is_dangerous_tool_call, - tool_call_input, - tool_call_name, - ) async def callback(tool_call: dict, options: list[dict]) -> dict: if is_dangerous_tool_call(tool_call): diff --git a/condor/controller_configs.py b/condor/controller_configs.py new file mode 100644 index 000000000..c5bae6876 --- /dev/null +++ b/condor/controller_configs.py @@ -0,0 +1,51 @@ +"""Controller-config normalization shared by every surface (ARCH-190). + +Pure helpers for cleaning a controller config before it is saved to the +backend. Moved here from ``handlers/bots/_shared.py`` so the web dashboard +(``condor/web/routes/bots.py``) no longer imports from the Telegram handlers +package; that module re-exports these names for its Telegram callers. +""" + +from __future__ import annotations + +import re +from typing import Any, Dict + +# Matches stringified enum values like "PositionMode.ONEWAY", "OrderType.LIMIT", +# "TradeType.BUY" or the colon variant "PositionMode:ONEWAY". The class name is +# PascalCase and the member starts with an uppercase letter, which avoids matching +# legitimate values such as trading times ("09:30") or token symbols ("USDC.e"). +_ENUM_STR_RE = re.compile(r"^[A-Z][A-Za-z0-9_]*[.:][A-Z][A-Za-z0-9_]*$") + + +def normalize_enum_value(value: Any) -> Any: + """Strip the enum-class prefix from stringified enum values. + + Controller config templates expose enum defaults as strings like + "PositionMode.ONEWAY". Saving them verbatim breaks downstream controller + validation, which expects the plain member name ("ONEWAY"). This recurses + into dicts and lists so nested config values are normalized too. + """ + if isinstance(value, str) and _ENUM_STR_RE.match(value): + return re.split(r"[.:]", value, maxsplit=1)[1] + if isinstance(value, dict): + return {k: normalize_enum_value(v) for k, v in value.items()} + if isinstance(value, list): + return [normalize_enum_value(v) for v in value] + return value + + +def clean_config_for_save(config: Dict[str, Any]) -> Dict[str, Any]: + """Strip internal fields and normalize stringified enum values before saving. + + The backend adds _config_name to YAML files, but Pydantic models with + extra='forbid' reject it on reload. Strip any underscore-prefixed keys + that aren't part of the controller config schema. + + Enum fields (e.g. position_mode, side, take_profit_order_type) can arrive as + prefixed strings like "PositionMode.ONEWAY"; normalize them to the plain + member name so deploy/backtest validation accepts them. + """ + return { + k: normalize_enum_value(v) for k, v in config.items() if not k.startswith("_") + } diff --git a/condor/fetchers/__init__.py b/condor/fetchers/__init__.py index e305cf52c..9d4ff7927 100644 --- a/condor/fetchers/__init__.py +++ b/condor/fetchers/__init__.py @@ -21,7 +21,7 @@ above ``_SNAPSHOT_TTL``). The aggregate ``fetch_all_bot_performance`` returns is therefore shared between callers and must be treated as read-only. - - No imports from handlers/ or condor.web/ (prevents circular deps). + - No handlers/ or condor.web imports (prevents circular deps). - Keep thin: call client method, light transform, return. Importing: diff --git a/condor/llm/__init__.py b/condor/llm/__init__.py new file mode 100644 index 000000000..76d71e0ea --- /dev/null +++ b/condor/llm/__init__.py @@ -0,0 +1,14 @@ +"""Platform-neutral LLM/model-catalog foundations (ARCH-190). + +Everything here answers "which models can Condor run, and are they ready?" for +every surface — the Telegram bot, the web dashboard, the setup wizard and the +MCP tools. Nothing in this package may import from ``handlers/`` or +``condor.web``: the layering is handlers -> condor, never the reverse. + +Modules: +- :mod:`condor.llm.options` — the agent/model catalog (``AGENT_OPTIONS``) and + the resolved default (``DEFAULT_AGENT``). +- :mod:`condor.llm.readiness` — "can this machine actually run that model?". +- :mod:`condor.llm.openrouter_models` — the OpenRouter catalog fetch. +- :mod:`condor.llm.custom_models` — OpenAI-compatible custom endpoints. +""" diff --git a/condor/llm/custom_models.py b/condor/llm/custom_models.py new file mode 100644 index 000000000..b8f212d35 --- /dev/null +++ b/condor/llm/custom_models.py @@ -0,0 +1,344 @@ +"""Custom OpenAI-compatible provider model fetcher. + +Hits GET {base_url}/models with an optional Bearer key — the standard model +list endpoint every OpenAI-compatible API exposes (Venice AI, Together, +Fireworks, vLLM, TGI, ...). A successful fetch doubles as validation of both +the URL and the API key during the custom-endpoint setup flow. +""" + +from __future__ import annotations + +import asyncio +import hashlib +import ipaddress +import logging +import os +import re +import socket +from urllib.parse import urlparse + +import aiohttp + +log = logging.getLogger(__name__) + +# Per-candidate budget. Candidates are probed concurrently, so this is also +# roughly the worst case for the whole fetch — the user is staring at a +# "validating..." message the entire time. +FETCH_TIMEOUT_SECONDS = 8 + +# Hosts that hand out cloud instance credentials to anything that can issue an +# HTTP GET. Always refused: no legitimate model endpoint lives here. +BLOCKED_HOSTNAMES = frozenset( + { + "metadata.google.internal", + "metadata.goog", + "instance-data", + } +) + +# Opt-in for multi-tenant deployments: also refuse loopback and RFC1918 +# targets. Off by default because pointing at a local Ollama/vLLM/LM Studio +# server is the single most common use of this feature. +_BLOCK_PRIVATE_ENV = "CUSTOM_LLM_BLOCK_PRIVATE_URLS" + +# /models entries whose declared type isn't one of these are not chat models. +# Providers disagree on the field name (`type` on Venice/Together, `object` on +# stock OpenAI) and on the vocabulary, so accept every spelling of "text in, +# text out" we've seen and treat other declared values as non-chat. +CHAT_MODEL_TYPES = frozenset({"text", "chat", "language", "llm"}) + +# Values that technically occupy the type field but say nothing about +# modality — stock OpenAI stamps object="model" on embeddings, Whisper and +# DALL·E alike. Treat these as "not declared" and fall through to the id. +UNINFORMATIVE_MODEL_TYPES = frozenset({"model", "list", "object"}) + +# Fallback for providers that return no type at all: drop ids that are +# unmistakably a non-chat modality. +_NON_CHAT_MARKERS = ( + "embed", + "rerank", + "whisper", + "tts", + "text-to-speech", + "speech", + "voice", + "moderation", + "guard", + "upscal", + "stable-diffusion", + "flux", + "dall-e", + "sdxl", + "image", + "vision-encoder", + "clip", + "ocr", +) + + +# urlparse is happy to call "not a url at all" a netloc, so the hostname needs +# its own shape check: dotted labels, or an IPv6 literal in brackets. +_HOSTNAME_RE = re.compile( + r"^(?:[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*\.?|\[[0-9A-Fa-f:.]+\])$" +) + + +class CustomProviderError(Exception): + """Raised when the endpoint can't be validated. Message is user-facing.""" + + +def _block_private() -> bool: + return os.environ.get(_BLOCK_PRIVATE_ENV, "").strip().lower() in ( + "1", + "true", + "yes", + "on", + ) + + +def _reject_if_blocked(host: str) -> None: + """Raise ValueError when `host` points somewhere we refuse to fetch from. + + Link-local addresses (the 169.254.169.254 cloud metadata range) are always + refused. Loopback and private ranges are allowed by default so local model + servers work, and can be refused too by setting + CUSTOM_LLM_BLOCK_PRIVATE_URLS=true on shared/multi-tenant deployments. + + Only IP literals are checked unless the strict flag is set — resolving + every hostname would add latency and would still be defeatable by DNS + rebinding, so this is a guard against casual probing, not a security + boundary. + """ + lowered = host.lower().rstrip(".") + if lowered in BLOCKED_HOSTNAMES: + raise ValueError(f"'{host}' is not an allowed endpoint host.") + + strict = _block_private() + addresses: list[str] = [] + try: + ipaddress.ip_address(lowered.strip("[]")) + addresses = [lowered.strip("[]")] + except ValueError: + if strict: + try: + addresses = [info[4][0] for info in socket.getaddrinfo(lowered, None)] + except OSError: + # Unresolvable — let the actual request produce the error + return + + for raw in addresses: + try: + ip = ipaddress.ip_address(raw) + except ValueError: + continue + if ip.is_link_local: + raise ValueError( + f"'{host}' resolves to a link-local address, which is blocked " + "(cloud instance metadata lives there)." + ) + if strict and (ip.is_loopback or ip.is_private or ip.is_reserved): + raise ValueError( + f"'{host}' resolves to a private address and " + f"{_BLOCK_PRIVATE_ENV} is enabled." + ) + + +def normalize_base_url(raw: str) -> str: + """Coerce user input into a usable OpenAI-compatible base URL. + + Adds https:// when no scheme is given and strips trailing slashes and + endpoint suffixes users commonly paste (/chat/completions, /models). + Raises ValueError for input that can't be a URL or that targets a + blocked host. + """ + url = raw.strip() + if not url: + raise ValueError("URL is empty.") + if "://" not in url: + url = f"https://{url}" + url = url.rstrip("/") + + try: + parsed = urlparse(url) + hostname = parsed.hostname + except ValueError: + raise ValueError(f"'{raw.strip()}' is not a valid http(s) URL.") from None + if ( + parsed.scheme not in ("http", "https") + or not parsed.netloc + or not hostname + or not _HOSTNAME_RE.match(hostname) + ): + raise ValueError(f"'{raw.strip()}' is not a valid http(s) URL.") + _reject_if_blocked(hostname) + + for suffix in ("/chat/completions", "/completions", "/models"): + if url.endswith(suffix): + url = url[: -len(suffix)] + break + return url.rstrip("/") + + +def _is_chat_model(entry: dict) -> bool: + """Best-effort guess at whether a /models entry can serve chat completions. + + Providers return embeddings, rerankers, TTS and image models from the same + endpoint. Picking one of those succeeds here and then fails at prompt time + with an opaque upstream error, so filter them out while we still have the + metadata to do it. + """ + declared = entry.get("type") or entry.get("object") + if isinstance(declared, str) and declared.strip(): + normalized = declared.strip().lower() + if normalized not in UNINFORMATIVE_MODEL_TYPES: + return normalized in CHAT_MODEL_TYPES + + capabilities = entry.get("capabilities") + if isinstance(capabilities, dict): + for flag in ("chat", "chat_completions", "supportsChatCompletions"): + if flag in capabilities: + return bool(capabilities[flag]) + + model_id = entry.get("id", "").lower() + return not any(marker in model_id for marker in _NON_CHAT_MARKERS) + + +def _extract_model_ids(payload: object, chat_only: bool = True) -> list[str]: + """Pull model ids from an OpenAI-format /models response. + + Expects {"data": [{"id": "..."}, ...]}. Returns sorted, deduped ids; + empty list when the payload doesn't match the expected shape. With + ``chat_only`` (the default) non-chat modalities are dropped. + """ + if not isinstance(payload, dict): + return [] + data = payload.get("data") + if not isinstance(data, list): + return [] + ids = { + entry["id"].strip() + for entry in data + if isinstance(entry, dict) + and isinstance(entry.get("id"), str) + and entry["id"].strip() + and (not chat_only or _is_chat_model(entry)) + } + return sorted(ids, key=str.lower) + + +async def _probe( + session: aiohttp.ClientSession, candidate: str, headers: dict +) -> tuple[list[str], CustomProviderError | None]: + """Fetch one candidate's model list. Returns (models, error).""" + models_url = f"{candidate}/models" + try: + async with session.get(models_url, headers=headers) as resp: + if resp.status in (401, 403): + return [], CustomProviderError( + f"the provider at {candidate} rejected the API key " + f"(HTTP {resp.status}). Check the key and try again." + ) + if resp.status == 404: + return [], CustomProviderError( + f"no /models endpoint at {candidate} (HTTP 404). " + "Check the base URL — it usually ends in /v1." + ) + if resp.status != 200: + return [], CustomProviderError( + f"the provider at {candidate} returned HTTP {resp.status}." + ) + payload = await resp.json(content_type=None) + except asyncio.CancelledError: + raise + except Exception as e: + return [], CustomProviderError( + f"could not reach {models_url} ({type(e).__name__}). " + "Check the URL and that the server is up." + ) + + model_ids = _extract_model_ids(payload) + if not model_ids: + # An endpoint that answered but listed only embeddings/image models is + # a different failure from one that isn't OpenAI-shaped at all. + if _extract_model_ids(payload, chat_only=False): + return [], CustomProviderError( + f"{models_url} returned models, but none of them are chat " + "models (only embeddings/image/audio)." + ) + return [], CustomProviderError( + f"{models_url} responded but did not return an " + "OpenAI-compatible model list." + ) + return model_ids, None + + +async def fetch_models(base_url: str, api_key: str = "") -> tuple[str, list[str]]: + """Validate an OpenAI-compatible endpoint and return its chat model list. + + Probes {base_url}/models and {base_url}/v1/models concurrently — the second + covers users who pasted the host root (e.g. https://api.venice.ai instead + of https://api.venice.ai/api/v1) — and prefers the first candidate that + works. Returns (resolved_base_url, model_ids). + + Raises CustomProviderError with a user-facing message on any failure — + unreachable host, rejected key, or a non-OpenAI-shaped response. + """ + headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} + candidates = [base_url] + if not base_url.endswith("/v1"): + candidates.append(f"{base_url}/v1") + + timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_SECONDS) + async with aiohttp.ClientSession(timeout=timeout) as session: + results = await asyncio.gather( + *(_probe(session, candidate, headers) for candidate in candidates) + ) + + # Candidate order is preference order: the URL the user actually typed wins + for candidate, (model_ids, _) in zip(candidates, results): + if model_ids: + log.info( + "Custom provider validated: %s (%d chat models)", + candidate, + len(model_ids), + ) + return candidate, model_ids + + for _, error in results: + if error: + raise error + raise CustomProviderError("no reachable endpoint found.") + + +def model_token(model_id: str) -> str: + """Short stable token for a model id, for use in callback_data. + + Telegram caps callback_data at 64 bytes and model ids are unbounded. A + content-derived token also survives the model list being refetched between + render and tap, which a bare list index does not. + """ + return hashlib.blake2s(model_id.encode("utf-8"), digest_size=4).hexdigest() + + +def find_by_token(model_ids: list[str], token: str) -> str | None: + """Resolve a token produced by :func:`model_token` back to its model id.""" + for model_id in model_ids: + if model_token(model_id) == token: + return model_id + return None + + +def format_model_label(model_id: str, limit: int = 38) -> str: + """Short label for inline keyboard buttons. + + Truncates through the middle: for ids like + ``meta-llama/Llama-3.3-70B-Instruct-Turbo-Free`` the tail is what + distinguishes one variant from another, so cutting it off is exactly the + wrong end to lose. + """ + if len(model_id) <= limit: + return model_id + keep = limit - 1 # room for the ellipsis + head = (keep + 1) // 2 + tail = keep - head + return f"{model_id[:head]}…{model_id[-tail:]}" diff --git a/condor/llm/openrouter_models.py b/condor/llm/openrouter_models.py new file mode 100644 index 000000000..8a7b17b58 --- /dev/null +++ b/condor/llm/openrouter_models.py @@ -0,0 +1,119 @@ +"""OpenRouter model catalog fetcher. + +Hits GET https://openrouter.ai/api/v1/models (public, unauthenticated) and +filters to entries that advertise `tools` in their supported_parameters. +Condor's whole architecture depends on tool-calling, so models without it +are hidden from the picker. + +Results are cached for an hour so paginating the picker doesn't refetch. +""" + +from __future__ import annotations + +import logging +import time +from dataclasses import dataclass + +import aiohttp + +log = logging.getLogger(__name__) + +OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" +CACHE_TTL_SECONDS = 3600 + + +@dataclass(frozen=True) +class OpenRouterModel: + slug: str # e.g. "anthropic/claude-sonnet-4-5" + name: str # human-friendly name from the API + context_length: int # tokens + prompt_price: float # USD per 1M input tokens, 0 if free + completion_price: float + + +_cache: tuple[float, list[OpenRouterModel]] | None = None + + +def _parse_price(value: object) -> float: + """OpenRouter returns prices as strings in USD per token. Convert to per-1M.""" + try: + return float(value) * 1_000_000 # type: ignore[arg-type] + except (TypeError, ValueError): + return 0.0 + + +def _model_supports_tools(entry: dict) -> bool: + params = entry.get("supported_parameters") or [] + return "tools" in params + + +async def fetch_models(force_refresh: bool = False) -> list[OpenRouterModel]: + """Return the list of OpenRouter models that support tool-calling. + + Sorted by provider then model id for predictable pagination. + """ + global _cache + + if not force_refresh and _cache is not None: + cached_at, models = _cache + if time.monotonic() - cached_at < CACHE_TTL_SECONDS: + return models + + timeout = aiohttp.ClientTimeout(total=10) + try: + async with aiohttp.ClientSession(timeout=timeout) as session: + async with session.get(OPENROUTER_MODELS_URL) as resp: + resp.raise_for_status() + payload = await resp.json() + except Exception as e: + log.warning("Failed to fetch OpenRouter models: %s", e) + if _cache is not None: + return _cache[1] # serve stale cache on failure + return [] + + raw = payload.get("data") or [] + models: list[OpenRouterModel] = [] + for entry in raw: + if not isinstance(entry, dict): + continue + slug = entry.get("id") + if not isinstance(slug, str) or not slug or slug.startswith("~"): + continue # skip canonical aliases like "~anthropic/claude-haiku-latest" + if not _model_supports_tools(entry): + continue + pricing = entry.get("pricing") or {} + models.append( + OpenRouterModel( + slug=slug, + name=str(entry.get("name") or slug), + context_length=int(entry.get("context_length") or 0), + prompt_price=_parse_price(pricing.get("prompt")), + completion_price=_parse_price(pricing.get("completion")), + ) + ) + + models.sort(key=lambda m: m.slug.lower()) + _cache = (time.monotonic(), models) + log.info("Fetched %d OpenRouter models with tool support", len(models)) + return models + + +def format_button_label(model: OpenRouterModel) -> str: + """Short label for inline keyboard buttons. Telegram max ~30 chars looks good.""" + label = model.name or model.slug + if len(label) > 38: + label = label[:35] + "..." + return label + + +def find_model_by_slug( + models: list[OpenRouterModel], slug: str +) -> OpenRouterModel | None: + """Case-insensitive exact-match lookup by slug.""" + target = slug.strip().lower() + if not target: + return None + for m in models: + if m.slug.lower() == target: + return m + return None diff --git a/condor/llm/options.py b/condor/llm/options.py new file mode 100644 index 000000000..527b437ae --- /dev/null +++ b/condor/llm/options.py @@ -0,0 +1,83 @@ +"""The agent/model catalog and the resolved default (ARCH-190). + +One catalog (``AGENT_OPTIONS``) for every surface that offers a model — the +Telegram menu, the web dashboard's dropdown, the setup wizard. Moved here from +``handlers/agents/_shared.py``, which re-exports these names for its Telegram +callers. +""" + +from __future__ import annotations + +import os +from typing import Any + +from condor.memory.paths import CHAT_SLUG + + +def _chat_agent(): + """The default agent's record — Condor's ``agents/condor/AGENT.md``. + + There is one loader and one frontmatter schema (FEAT-033): the chat is read + by ``AgentStore`` exactly like a specialist is. Returns ``None`` only if the + file is missing or unreadable, which callers treat as "no instructions" + rather than as a startup failure. + """ + from condor.agents.agent import AgentStore + + return AgentStore().get(CHAT_SLUG) + + +AGENT_OPTIONS: dict[str, dict[str, Any]] = { + "claude-code": {"label": "Claude Code"}, + "claude-acp:opus": {"label": "Claude (ACP) — Opus"}, + "claude-acp:sonnet": {"label": "Claude (ACP) — Sonnet"}, + "gemini": {"label": "Gemini CLI"}, + "copilot": {"label": "GitHub Copilot CLI"}, + "codex": {"label": "ChatGPT Codex"}, + "ollama:": {"label": "Ollama — Default Model"}, + "lmstudio:": {"label": "LM Studio — Default Model"}, + # Sentinels — these open a picker instead of being a selectable model, so + # they are NOT valid agent keys. `picker` marks them for surfaces that + # render AGENT_OPTIONS as a flat list of choices (the web dashboard's model + # dropdown), which would otherwise offer a key that fails at session start. + # + # OpenRouter: stored agent_llm becomes "openrouter:". + "openrouter:": {"label": "OpenRouter — Pick Model", "picker": True}, + # Custom endpoints: the user's saved OpenAI-compatible endpoints live in + # preferences (condor/preferences.py, shared with the web dashboard) and + # the stored agent_llm becomes "custom@:". + "custom:": {"label": "Custom — OpenAI-compatible API", "picker": True}, +} + + +# What a fresh install should pick unless it has a reason not to: the ACP bridge +# is the best-supported path, and Sonnet is the balance of capability and cost +# most users want. Surfaces that rank the catalog (the `make pick-model` wizard) +# float this key to the top and mark it; it is deliberately NOT part of the +# `label`, which is reused as a plain display name all over the bot. +RECOMMENDED_AGENT = "claude-acp:sonnet" + + +def selectable_agent_options() -> dict[str, dict[str, Any]]: + """AGENT_OPTIONS minus the picker sentinels — every key here is startable.""" + return {k: v for k, v in AGENT_OPTIONS.items() if not v.get("picker")} + + +def _default_agent() -> str: + """Resolve the fallback agent_key for users who haven't picked a model. + + Precedence: ``CONDOR_DEFAULT_AGENT`` env > Condor's ``AGENT.md`` frontmatter + ``agent_key`` > ``"claude-code"``. A user's own /agent → Change LLM choice + (``agent_llm``) still overrides this at runtime; this is only the default. + Examples for the frontmatter / env: ``claude-code``, ``claude-acp:opus``, + ``ollama:qwen3:32b``. + """ + agent = _chat_agent() + return ( + os.environ.get("CONDOR_DEFAULT_AGENT") + or (agent.agent_key if agent else "") + or "claude-code" + ) + + +DEFAULT_AGENT = _default_agent() diff --git a/condor/llm/readiness.py b/condor/llm/readiness.py new file mode 100644 index 000000000..340177322 --- /dev/null +++ b/condor/llm/readiness.py @@ -0,0 +1,298 @@ +"""One answer to "can Condor actually run this model right now?". + +Every surface that offers a model — the ``get_available_models`` MCP tool behind +the agent_builder skill, and the setup wizard (``condor/setup_llm.py``) — asks +this module, so they can never disagree about what a machine can run. The probes +source their commands from :data:`condor.acp.ACP_COMMANDS` and their base URLs +from :data:`condor.acp.pydantic_ai_client.DEFAULT_BASE_URLS`, so adding a +provider there lights it up here for free. + +Three states, never two: a CLI bridge that is installed but whose login cannot be +proven is ``UNVERIFIED``, not ready and not missing. The login markers below are +undocumented paths that upstream CLIs may move, so they only ever downgrade a row +from *ready* to *unverified* — never to *missing*, and never enough to refuse a +pick. + +Read-only and side-effect-free. Never returns a key value, and an unreachable +local server is a normal state (the user simply isn't running it), reported as +``MISSING`` with the command that starts it, never raised. +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +import sys +from dataclasses import dataclass, field +from pathlib import Path +from typing import Iterable + +import httpx + +from condor.acp import ACP_COMMANDS +from condor.acp.pydantic_ai_client import DEFAULT_BASE_URLS + +READY, UNVERIFIED, MISSING = "ready", "unverified", "missing" + +# Cloud providers Condor can address as ":", mapped to the env +# var holding each key. Presence of the key = the provider is usable. Sourced +# here (not hardcoded per surface) so the set stays in one place. +CLOUD_KEY_ENVS = { + "openrouter": "OPENROUTER_API_KEY", + "openai": "OPENAI_API_KEY", + "anthropic": "ANTHROPIC_API_KEY", + "groq": "GROQ_API_KEY", + "google": "GOOGLE_API_KEY", +} + +# Local, keyless OpenAI-compatible servers. Base URLs come from the resolver's +# own defaults (DEFAULT_BASE_URLS) so they can never drift from what actually +# runs the model at request time. +LOCAL_PREFIXES = ("ollama", "lmstudio") + +# What to run once, interactively, to log a bridge in. Not run from here: the +# wizard reports and instructs, it never drives someone else's OAuth flow. +_LOGIN_COMMANDS = { + "claude-code": "claude", + "claude-acp": "claude", + "gemini": "npx @google/gemini-cli", + "copilot": "npx @github/copilot", + "codex": "codex", +} + +# Bridges launched as a bare command still come from npm; name the package so a +# missing one is actionable instead of just absent. +_BARE_INSTALL = { + "claude-agent-acp": "npm install -g @agentclientprotocol/claude-agent-acp", +} + +_START_COMMANDS = { + "ollama": "start it with `ollama serve`", + "lmstudio": "start LM Studio and enable its local server", +} + + +@dataclass(frozen=True) +class Readiness: + """What a menu row needs: a state, a sentence, and any models behind it.""" + + state: str # READY | UNVERIFIED | MISSING + detail: str # human sentence; when not READY it names the fix + models: list[str] = field(default_factory=list) # local server models; [] elsewhere + + @property + def usable(self) -> bool: + """False only for MISSING — an unproven login must never block a pick.""" + return self.state != MISSING + + +def npx_packages_installed() -> set[str]: + """npm packages resolvable WITHOUT a network install: local, global, npx cache. + + Mirrors how npx resolves a package before falling back to downloading it. + ``shutil.which("npx")`` says nothing about the package — npx exists whenever + Node does — so probing the launcher would mark every ``npx``-run bridge + available on any machine with Node. + """ + roots = [Path.cwd() / "node_modules"] + npm = shutil.which("npm") + if npm: + try: + proc = subprocess.run( + [npm, "root", "-g"], capture_output=True, text=True, timeout=10 + ) + if proc.returncode == 0 and proc.stdout.strip(): + roots.append(Path(proc.stdout.strip())) + except (OSError, subprocess.SubprocessError): + pass # npm unusable = nothing installed via it; local + cache still count + roots.extend((Path.home() / ".npm" / "_npx").glob("*/node_modules")) + + installed: set[str] = set() + for root in roots: + if not root.is_dir(): + continue + for entry in root.iterdir(): + if entry.name.startswith("@"): + installed.update(f"{entry.name}/{sub.name}" for sub in entry.iterdir()) + else: + installed.add(entry.name) + return installed + + +def _keychain_has(service: str) -> bool | None: + """macOS keychain lookup: True found, False absent, None cannot tell.""" + security = shutil.which("security") + if not security: + return None + try: + proc = subprocess.run( + [security, "find-generic-password", "-s", service], + capture_output=True, + text=True, + timeout=5, + ) + except (OSError, subprocess.SubprocessError): + return None + return proc.returncode == 0 + + +def acp_login_state(base: str) -> bool | None: + """Has this bridge's interactive login already happened? + + ``True`` a credential exists, ``False`` none was found where that CLI keeps + one, ``None`` this bridge has no marker we can read (GitHub Copilot) or the + machine could not be asked. Heuristic by construction: a credentials file + proves a login happened, not that the token is still live, and its absence + proves nothing on a machine using another auth path — which is why callers + treat False and None alike as *unverified*, never as *not installed*. + """ + home = Path.home() + if base in ("claude-code", "claude-acp"): + if (home / ".claude" / ".credentials.json").exists(): + return True + if sys.platform == "darwin": # Claude Code stores it in the login keychain + return _keychain_has("Claude Code-credentials") + return False + if base == "gemini": + if (home / ".gemini" / "oauth_creds.json").exists(): + return True + return bool(os.environ.get("GEMINI_API_KEY")) or False + if base == "codex": + return (home / ".codex" / "auth.json").exists() + return None # copilot and anything new: no documented on-disk marker + + +def install_command(cmd: str) -> str: + """How to install the CLI behind an ACP command line.""" + parts = cmd.split() + if len(parts) > 1 and parts[0] == "npx": + return f"npm install -g {parts[1]}" + if parts and parts[0] in _BARE_INSTALL: + return _BARE_INSTALL[parts[0]] + return f"install `{cmd.split()[0] if parts else cmd}` and put it on PATH" + + +def acp_bridges() -> list[dict]: + """ACP CLI bridges: whether each is installed, and whether it is logged in. + + A bridge launched as ``npx …`` is available only if the PACKAGE is + already installed; a bare command is available if it is on PATH. ``available`` + means "launchable" — a bridge additionally needs its own interactive login, + reported separately as ``logged_in`` (``None`` = no marker to read). + """ + npx_packages = npx_packages_installed() + out: list[dict] = [] + for base, cmd in ACP_COMMANDS.items(): + parts = cmd.split() + if not parts: + available = False + elif parts[0] == "npx": + available = parts[1] in npx_packages if len(parts) > 1 else False + else: + available = shutil.which(parts[0]) is not None + out.append( + { + "agent_key": base, + "command": cmd, + "available": available, + "logged_in": acp_login_state(base) if available else None, + } + ) + return out + + +async def local_servers() -> dict: + """Probe local OpenAI-compatible servers and list their loaded models. + + Unreachable servers are reported (``reachable: false``), not raised — the + user just isn't running that server. + """ + result: dict = {} + timeout = httpx.Timeout(connect=1.5, read=3.0, write=1.5, pool=1.5) + async with httpx.AsyncClient(timeout=timeout) as client: + for prefix in LOCAL_PREFIXES: + base = DEFAULT_BASE_URLS.get(prefix) + entry = {"base_url": base, "reachable": False, "models": []} + if base: + try: + resp = await client.get(f"{base.rstrip('/')}/models") + if resp.status_code == 200: + data = resp.json().get("data", []) + entry["reachable"] = True + entry["models"] = sorted( + m["id"] for m in data if isinstance(m, dict) and m.get("id") + ) + except Exception: + pass # not running = normal, leave reachable=False + result[prefix] = entry + return result + + +def _bridge_readiness(entry: dict) -> Readiness: + base, cmd = entry["agent_key"], entry["command"] + if not entry["available"]: + return Readiness(MISSING, f"not installed — {install_command(cmd)}") + login = _LOGIN_COMMANDS.get(base, cmd) + if entry.get("logged_in") is True: + return Readiness(READY, "installed and logged in") + if entry.get("logged_in") is False: + return Readiness(UNVERIFIED, f"installed; run `{login}` once and log in") + return Readiness( + UNVERIFIED, + f"installed; login could not be verified — if your first chat " + f"fails, run `{login}` once and log in", + ) + + +def _local_readiness(prefix: str, entry: dict) -> Readiness: + if not entry["reachable"]: + hint = _START_COMMANDS.get(prefix, "start the server") + return Readiness(MISSING, f"not reachable at {entry['base_url']} — {hint}") + models = list(entry["models"]) + return Readiness(READY, f"{len(models)} model(s) available", models) + + +def _key_readiness(base: str, env: dict[str, str]) -> Readiness: + var = CLOUD_KEY_ENVS[base] + if env.get(var) or os.environ.get(var): + return Readiness(READY, f"{var} is set") + if base == "openrouter": + return Readiness(MISSING, "needs an API key (you can enter one here)") + return Readiness(MISSING, f"needs {var} in .env") + + +async def probe_all( + bases: Iterable[str], env: dict[str, str] | None = None +) -> dict[str, Readiness]: + """Readiness for several provider bases, probing each source exactly once. + + ``env`` is passed in rather than read from the process so the setup wizard + can probe against a ``.env`` the bot has not loaded yet; ``os.environ`` is + still consulted as a fallback. + """ + env = env or {} + wanted = list(dict.fromkeys(bases)) + bridges: dict[str, dict] = {} + if any(b in ACP_COMMANDS for b in wanted): + bridges = {e["agent_key"]: e for e in acp_bridges()} + servers: dict = {} + if any(b in LOCAL_PREFIXES for b in wanted): + servers = await local_servers() + + out: dict[str, Readiness] = {} + for base in wanted: + if base in bridges: + out[base] = _bridge_readiness(bridges[base]) + elif base in servers: + out[base] = _local_readiness(base, servers[base]) + elif base in CLOUD_KEY_ENVS: + out[base] = _key_readiness(base, env) + else: + out[base] = Readiness(UNVERIFIED, "cannot be checked from here") + return out + + +async def probe(base: str, env: dict[str, str] | None = None) -> Readiness: + """Readiness for a single provider base (see :func:`probe_all`).""" + return (await probe_all([base], env))[base] diff --git a/condor/runtime/binding.py b/condor/runtime/binding.py index c4113e465..8d2388494 100644 --- a/condor/runtime/binding.py +++ b/condor/runtime/binding.py @@ -17,7 +17,9 @@ import logging from dataclasses import dataclass, field +from condor.llm import options as llm_options from condor.memory.paths import CHAT_SLUG +from condor.runtime import toolsets from condor.runtime.models import SessionSpec log = logging.getLogger(__name__) @@ -86,7 +88,6 @@ def resolve( Callers get a fully resolved toolset and identity and never re-derive either. """ from condor.agents.agent import Agent, AgentStore - from handlers.agents._shared import build_mcp_servers_for_session slug = spec.agent_slug or CHAT_SLUG agent = AgentStore().get(slug) @@ -116,7 +117,7 @@ def resolve( # in CONDOR_USER_ID/CONDOR_CHAT_ID, and argv beats env in the subprocess, so # a local fallback here would silently override the env one (SEC-180). effective_user_id, effective_chat_id = spec.effective_ids() - mcp_servers = build_mcp_servers_for_session( + mcp_servers = toolsets.build_mcp_servers_for_session( effective_user_id, effective_chat_id, user_data, @@ -153,13 +154,11 @@ def remember_model_choice(user_id: int | None, agent_slug: str, agent_key: str) delegate and loops pass concrete keys for their own plumbing reasons and must not rewrite anything. """ - from handlers.agents._shared import AGENT_OPTIONS - if not agent_key or not user_id: return # Drill-downs, not startable models: an agent_key of "openrouter:" would # fail at session start for everyone who inherited it. - if AGENT_OPTIONS.get(agent_key, {}).get("picker"): + if llm_options.AGENT_OPTIONS.get(agent_key, {}).get("picker"): return try: diff --git a/condor/runtime/channels.py b/condor/runtime/channels.py new file mode 100644 index 000000000..6869b8a5b --- /dev/null +++ b/condor/runtime/channels.py @@ -0,0 +1,48 @@ +"""Delivery channels for pending trade confirmations (ARCH-190). + +The pending approval itself lives in :mod:`condor.runtime.confirmations`; a +channel only renders it somewhere a human can answer. ``TelegramChannel`` moved +here from ``handlers/agents/confirmation.py`` so the runtime (consult, +delegate) no longer imports from the hot-reloaded handlers package; that module +re-exports it for its Telegram callers. +""" + +from __future__ import annotations + +from telegram import Bot, InlineKeyboardButton, InlineKeyboardMarkup + +from condor.runtime.confirmations import PendingConfirmation + + +class TelegramChannel: + """Renders a pending confirmation as an inline keyboard in a chat.""" + + def __init__(self, bot: Bot, chat_id: int): + self._bot = bot + self._chat_id = chat_id + + async def deliver(self, pending: PendingConfirmation) -> None: + keyboard = InlineKeyboardMarkup( + [ + [ + InlineKeyboardButton( + "Approve", callback_data=f"agent:confirm_trade:{pending.id}" + ), + InlineKeyboardButton( + "Reject", callback_data=f"agent:reject_trade:{pending.id}" + ), + ] + ] + ) + # Naming the asker is not decoration: one chat can talk to several + # agents on several servers, and "Approve this action?" on its own does + # not say whose action, or where it lands. + asked_by = f"\nRequested by {pending.origin}\n" if pending.origin else "" + await self._bot.send_message( + chat_id=self._chat_id, + text=( + f"Trade Confirmation\n{asked_by}\n" + f"{pending.summary}\n\nApprove this action?" + ), + reply_markup=keyboard, + ) diff --git a/condor/runtime/confirmations.py b/condor/runtime/confirmations.py index d18821e6e..01d3fc545 100644 --- a/condor/runtime/confirmations.py +++ b/condor/runtime/confirmations.py @@ -25,6 +25,7 @@ from enum import Enum from typing import Any, Protocol +from condor.runtime import danger from condor.runtime.timeouts import TIMEOUTS from condor.telemetry import taps as telemetry_taps @@ -352,21 +353,19 @@ def build_permission_callback( async def permission_callback( tool_call: dict[str, Any], options: list[dict[str, Any]] ) -> dict[str, Any]: - from handlers.agents._shared import BLOCKED_TOOLS, is_dangerous_tool_call - from handlers.agents.confirmation import format_tool_summary - tool_name = tool_call.get("tool", "") or tool_call.get("title", "") - if tool_name in BLOCKED_TOOLS: + # Read through the module so a monkeypatched BLOCKED_TOOLS is honored. + if tool_name in danger.BLOCKED_TOOLS: log.warning("Blocked tool %s for session %s", tool_name, session_key) return CANCELLED - if not is_dangerous_tool_call(tool_call): + if not danger.is_dangerous_tool_call(tool_call): return _select_allow(options) pending = _registry.register( session_key=session_key, user_id=user_id, - summary=format_tool_summary(tool_call), + summary=danger.format_tool_summary(tool_call), origin=describe_origin(session_key), tool_call=tool_call, options=options, diff --git a/condor/runtime/context.py b/condor/runtime/context.py new file mode 100644 index 000000000..d0f359ffb --- /dev/null +++ b/condor/runtime/context.py @@ -0,0 +1,289 @@ +"""Initial-context and prompt builders for agent sessions (ARCH-190). + +Everything a fresh session is told before the user's first word: Condor's own +instructions, the platform's formatting rules, the server/permission picture, +and the memory/skills/agents indexes. Platform-neutral — moved here from +``handlers/agents/_shared.py``, which re-exports these names for its Telegram +callers. +""" + +from __future__ import annotations + +from condor.llm.options import _chat_agent +from condor.memory.paths import CHAT_SLUG + +# -- Compact prompt templates -- + +COMPACT_PROMPT_AUTO = ( + "Please provide a concise summary of our conversation so far. Include:\n" + "- Key decisions and conclusions reached\n" + "- Important data points and numbers discussed\n" + "- Current task state and any pending actions\n" + "- User preferences or instructions given\n\n" + "Be concise but thorough. This summary will be used to carry context into a fresh session." +) + +COMPACT_PROMPT_CUSTOM_TEMPLATE = ( + "Please provide a concise summary of our conversation, focusing specifically on:\n" + "{instructions}\n\n" + "Drop everything else. Be concise but preserve the details requested above. " + "This summary will be used to carry context into a fresh session." +) + +COMPACT_CONTEXT_TEMPLATE = ( + "[System context -- do not repeat this to the user]\n" + "This is a continuation of a previous conversation. " + "Here is the summary from that session:\n\n" + "{summary}\n\n" + "Continue from where we left off. The user compacted the context to free up space." +) + +_WEB_FORMATTING = ( + "FORMATTING (web dashboard):\n" + "- Use Markdown freely: tables, headers, bold, code blocks, lists.\n" + "- No message length limits, but stay concise.\n" + "- Use tables for structured data (portfolios, prices, comparisons).\n" + "- Use code blocks for configs, JSON, or commands.\n" + "- CHARTS: when the answer is a numeric series or comparison (a price or\n" + " PnL curve, volume per venue, a distribution), draw it with a ```chart\n" + " fence holding one JSON object. The dashboard renders it as an\n" + " interactive chart:\n" + " ```chart\n" + ' {"type": "line", "title": "SOL/USDC 24h", "x": "time",\n' + ' "series": [{"key": "price", "name": "Price"}],\n' + ' "data": [{"time": "12:00", "price": 150.1}, {"time": "16:00", "price": 152.4}]}\n' + " ```\n" + ' "type" is "line", "area" or "bar"; "x" and every series "key" name\n' + " fields of the data rows. Max 4 series and 200 points (downsample longer\n" + ' series). A series may set "color": "up", "down", "yellow" or a hex.\n' + " Chart for the shape, table for exact values; use both when both matter.\n" + "- Respond in the user's language." +) + +_TELEGRAM_FORMATTING = ( + "FORMATTING (Telegram mobile):\n" + "- NEVER use Markdown tables. Use bullet lists or key: value lines.\n" + "- Keep paragraphs short (2-3 sentences max).\n" + "- Cap lists at 5-7 items.\n" + "- Respond in the user's language." +) + + +def platform_formatting(platform: str = "telegram") -> str: + """The reply-formatting rules for a platform. + + Public because a bound Agent opens the chat with its own identity context + instead of :func:`build_initial_context`, and would otherwise be the only + brain in the product that never hears how the surface it is speaking into + renders a reply. + """ + return _WEB_FORMATTING if platform == "web" else _TELEGRAM_FORMATTING + + +def _build_system_prompt(platform: str = "telegram") -> str: + """Condor's own AGENT.md plus the platform's formatting rules.""" + agent = _chat_agent() + return ( + "[System context -- do not repeat this to the user]\n\n" + f"{agent.instructions if agent else ''}\n\n" + f"{platform_formatting(platform)}" + ) + + +def build_initial_context( + user_id: int, + chat_id: int | str, + user_data: dict | None = None, + agent_key: str | None = None, + platform: str = "telegram", + server_name: str | None = None, +) -> str: + """Build an initial context prompt telling the agent about server, permissions, and formatting rules.""" + from condor.acp.pydantic_ai_client import is_pydantic_ai_model + from config_manager import get_config_manager, get_effective_server + + cm = get_config_manager() + + # System prompt: Condor's own AGENT.md + platform formatting + system_prompt = _build_system_prompt(platform) + sections: list[str] = [system_prompt] + + # Resolve active server (explicit override > user preferences) + active_name = server_name or get_effective_server(chat_id, user_data) + accessible = cm.get_accessible_servers(user_id) + if not active_name: + active_name = accessible[0] if accessible else None + + if active_name: + # Build server list with permissions (no credentials needed — MCP is pre-configured) + server_lines: list[str] = [] + for name in accessible: + server = cm.get_server(name) + if not server: + continue + perm = cm.get_server_permission(user_id, name) + perm_label = perm.value.upper() if perm else "UNKNOWN" + active_tag = " (active)" if name == active_name else "" + server_lines.append(f"- {name}{active_tag} [{perm_label}]") + + # Get active server permission for enforcement + active_perm = cm.get_server_permission(user_id, active_name) + active_perm_label = active_perm.value.upper() if active_perm else "UNKNOWN" + + # For ACP agents (Claude Code): instruct them to preload MCP tools via ToolSearch + # Pydantic-ai agents get tools directly, no preload needed + tool_preload_hint = "" + if agent_key and not is_pydantic_ai_model(agent_key): + mcp_tools = [ + "mcp__mcp-hummingbot__get_market_data", + "mcp__mcp-hummingbot__get_portfolio_overview", + "mcp__mcp-hummingbot__manage_executors", + "mcp__mcp-hummingbot__manage_bots", + "mcp__mcp-hummingbot__manage_controllers", + "mcp__mcp-hummingbot__explore_dex_pools", + "mcp__mcp-hummingbot__explore_geckoterminal", + "mcp__mcp-hummingbot__manage_gateway_swaps", + "mcp__mcp-hummingbot__manage_gateway_config", + "mcp__mcp-hummingbot__manage_gateway_container", + "mcp__mcp-hummingbot__search_history", + "mcp__mcp-hummingbot__set_account_position_mode_and_leverage", + "mcp__condor__manage_routines", + "mcp__condor__manage_servers", + "mcp__condor__get_user_context", + "mcp__condor__manage_trading_agent", + "mcp__condor__trading_agent_journal_read", + "mcp__condor__trading_agent_journal_write", + "mcp__condor__send_notification", + "mcp__condor__manage_memory", + "mcp__condor__manage_skill", + ] + tool_preload_hint = ( + "IMPORTANT: At the very start of the session (before your first response), " + "load ALL MCP tools in a single ToolSearch call:\n" + f'ToolSearch(query="select:{",".join(mcp_tools)}")\n' + "This avoids repeated ToolSearch calls that waste context tokens. " + "Do this silently without telling the user." + ) + + # Build server info section + server_info = [ + f"Active server: {active_name} (permission: {active_perm_label})", + "The MCP server is already connected to this server. Do NOT call configure_server — it is pre-configured.", + "", + ] + + if tool_preload_hint: + server_info.extend([tool_preload_hint, ""]) + + server_info.extend( + [ + "Available servers:", + *server_lines, + "", + "Server switching is not supported mid-session. If the user wants a different server, " + "tell them to start a new session with that server selected.", + "", + "Permission rules:", + f"- Your current permission on '{active_name}' is {active_perm_label}.", + "- OWNER: Full access including trading operations and server management.", + "- TRADER: Can trade, view balances, and manage own settings.", + "- Enforce the permission level for this server.", + ] + ) + + sections.append("\n".join(server_info)) + + # User memory index — what the chat assistant remembers about this user. This + # store is the chat's own (FEAT-003), not shared with the user's trading + # agents. Inject only the index; bodies are read on demand via + # manage_memory(action="read"). Nothing injected for new users. + try: + from condor.memory import MemoryStore + + memory_index = MemoryStore(user_id).list_index() + if memory_index: + sections.append( + "[USER MEMORY — what you remember about this user]\n" + "Consider this before responding. Read a full memory with " + 'manage_memory(action="read", name="..."). When you learn something ' + 'new and stable about the user, save it with manage_memory(action="write", ...).\n\n' + f"{memory_index}" + ) + except Exception: + pass # Memory is advisory — never block session start on it. + + # Skills index — read-only playbooks the chat assistant ships (know-how + + # steps). Skills are general to the assistant, not learned per user; inject + # only the index, bodies are read on demand via manage_skill(action="read"). + # Nothing injected when the assistant ships none. + try: + from condor.memory import SkillStore + + skills_index = SkillStore().list_index() + if skills_index: + sections.append( + "[SKILLS — check here BEFORE handling a known flow with raw tools]\n" + "Read-only playbooks shipped with the assistant. Before using raw " + "tools for a flow below, read its playbook with " + 'manage_skill(action="read", name="...") and follow the steps — ' + "don't re-derive or hand-roll what a playbook already covers.\n\n" + f"{skills_index}" + ) + except Exception: + pass # Skills are advisory — never block session start on them. + + # Agents index — domain Agents condor can consult (FEAT: coordinator model). + # condor delegates domain work via consult(...) instead of holding the domain's + # tools/context itself. EVERY agent is listed: one missing from this index can + # never be routed to. Nothing injected only when none exist. + try: + from condor.agents.agent import AgentStore + + # The coordinator is not among its own consultees (FEAT-033). + agents_index = AgentStore().list_index(exclude={CHAT_SLUG}) + if agents_index: + sections.append( + "[AGENTS — consult these BEFORE doing domain work with raw tools]\n" + "You are a coordinator. If a request falls in an agent's domain " + "below, delegate it with " + 'consult(agent="", task="...", context="...") instead of ' + "driving the domain's raw tools yourself — the agent has the focused " + "tools and domain memory. Relay a concise summary of its answer.\n\n" + f"{agents_index}" + ) + except Exception: + pass # The agents index is advisory — never block session start on it. + + return "\n\n".join(sections) + + +def build_agent_context( + agent: "Agent", # noqa: F821 — condor.agents.agent.Agent + user_id: int, + task: str, + context: str = "", +) -> str: + """Assemble the prompt for an Agent consult. + + Mirrors :func:`build_initial_context` but for a worker run: the Agent's own + instructions become the system prompt, its domain-scoped memory/skills indexes + are injected (keyed by the Agent slug, FEAT-003 — the shared brain), and the + consult task is appended last. Read on demand via manage_memory/manage_skill + inside the run. + + Those indexes come from :func:`~condor.memory.domain_context`, the same + builder ``binding.agent_identity_context`` composes, so this Agent carries + identical instructions about its own memory whether it is consulted or + chatted with (ARCH-099). + """ + from condor.memory import domain_context + + sections: list[str] = [agent.instructions] + sections.extend(domain_context(agent.slug, user_id)) + + consult = f"[CONSULT REQUEST]\n{task}" + if context: + consult += f"\n\n[CONTEXT FROM CONDOR]\n{context}" + sections.append(consult) + + return "\n\n".join(sections) diff --git a/condor/runtime/danger.py b/condor/runtime/danger.py new file mode 100644 index 000000000..fc2a7540a --- /dev/null +++ b/condor/runtime/danger.py @@ -0,0 +1,198 @@ +"""Security-critical tool-call classification (ARCH-190). + +Which tool calls need a human, which are refused outright, and how a pending +call is summarized for the confirmation prompt. This used to live in the +Telegram package (``handlers/agents/_shared.py`` / ``confirmation.py``), which +``main.py`` hot-reloads via watchfiles — so the runtime's trade gate depended +on reload timing. It is platform-neutral and belongs to the runtime; the +handlers modules re-export from here for their Telegram callers. + +Stdlib-only on purpose: everything that dispatches on a tool name (the danger +list, the risk gate, the confirmation summary) imports this module, so it must +never grow an import edge back into the runtime or the handlers. +""" + +from __future__ import annotations + +import json +from typing import Any + +# Tools that require user confirmation before execution +DANGEROUS_TOOLS = { + "place_order", + "manage_gateway_swaps", # execute action + "manage_gateway_clmm", # open/close position +} + +# Tools that are always blocked (RBAC bypass prevention) +BLOCKED_TOOLS: set[str] = set() + +# Actions within manage_executors that require confirmation +DANGEROUS_EXECUTOR_ACTIONS = {"create", "stop"} + +# Actions within manage_bots that deploy/mutate a live bot (status/logs/get_config +# are read-only and excluded). manage_controllers itself is excluded entirely — it +# only writes controller templates/saved configs, never a running bot (see its own +# tool docstring: "Does NOT affect running bots"). +DANGEROUS_BOT_ACTIONS = { + "deploy", + "stop_bot", + "stop_controllers", + "start_controllers", + "update_config", +} + +# Actions within manage_gateway_swaps that require confirmation +DANGEROUS_SWAP_ACTIONS = {"execute"} + +# Actions within manage_gateway_clmm that require confirmation +DANGEROUS_CLMM_ACTIONS = {"open_position", "close_position"} + + +def tool_call_name(tool_call: dict[str, Any]) -> str: + """The bare tool name, with any MCP prefix stripped. + + ACP names a tool by its wire name (``mcp__mcp-hummingbot__manage_bots``) + where the local surfaces use the bare one, so everything that dispatches on + a tool name — the danger list, the risk gate, the confirmation summary — + has to normalize identically or they disagree about the same call. + """ + raw_name = tool_call.get("tool", "") or tool_call.get("title", "") + return raw_name.rsplit("__", 1)[-1] if "__" in raw_name else raw_name + + +def tool_call_input(tool_call: dict[str, Any]) -> dict[str, Any] | None: + """A tool call's arguments as a mapping, or ``None`` when unreadable. + + ``None`` means "I cannot tell what this call does", and every caller has to + fail closed on it: an action-gated tool whose action can't be read is + treated as dangerous rather than waved through (SEC-093). Callers must not + reach for other spellings of the arguments — the ACP wire's ``rawInput`` is + translated once, at the boundary in :func:`condor.acp.client.normalize_tool_call`. + + A JSON string is parsed: OpenAI-compatible providers deliver tool arguments + that way. + """ + args = tool_call.get("input") + if isinstance(args, dict): + return args + if isinstance(args, str) and args.strip(): + try: + parsed = json.loads(args) + except json.JSONDecodeError: + return None + return parsed if isinstance(parsed, dict) else None + return None + + +def _has_dangerous_action( + tool_call: dict[str, Any], dangerous_actions: set[str] +) -> bool: + """Whether an action-gated tool call selects one of its dangerous actions. + + Fails closed (SEC-093): unreadable arguments, or a missing/non-string + ``action``, count as dangerous. ``action`` is required by every one of + these tools, so an unreadable one is never a benign read — it is a call we + failed to understand, and those belong in front of a human. + """ + input_data = tool_call_input(tool_call) + if input_data is None: + return True + action = input_data.get("action") + if not isinstance(action, str) or not action: + return True + return action in dangerous_actions + + +def is_dangerous_tool_call(tool_call: dict[str, Any]) -> bool: + """Check if a tool call requires user confirmation.""" + tool_name = tool_call_name(tool_call) + + # Direct dangerous tools + if tool_name in DANGEROUS_TOOLS: + # For manage_gateway_swaps, only "execute" action is dangerous + if tool_name == "manage_gateway_swaps": + return _has_dangerous_action(tool_call, DANGEROUS_SWAP_ACTIONS) + + # For manage_gateway_clmm, only open/close are dangerous + if tool_name == "manage_gateway_clmm": + return _has_dangerous_action(tool_call, DANGEROUS_CLMM_ACTIONS) + + return True + + # manage_executors with create/stop actions + if tool_name == "manage_executors": + return _has_dangerous_action(tool_call, DANGEROUS_EXECUTOR_ACTIONS) + + # manage_bots with deploy/stop/update actions (a bot deploy places real + # capital via a controller, a different path than manage_executors) + if tool_name == "manage_bots": + return _has_dangerous_action(tool_call, DANGEROUS_BOT_ACTIONS) + + return False + + +def format_tool_summary(tool_call: dict[str, Any]) -> str: + """Format a tool call into a human-readable summary for the confirmation message.""" + # The bare name, or an ACP call (``mcp__mcp-hummingbot__manage_bots``) would + # match none of the branches below and be approved as an opaque string. + tool_name = tool_call_name(tool_call) or "Unknown" + input_data = tool_call_input(tool_call) + if input_data is None: + # The gate sends unreadable calls here on purpose (SEC-093). Say so, + # rather than rendering a row of "?" that looks like a parsed call. + return f"{tool_name} (arguments could not be read)" + + if tool_name == "place_order": + side = input_data.get("trade_type", "?") + pair = input_data.get("trading_pair", "?") + amount = input_data.get("amount", "?") + order_type = input_data.get("order_type", "MARKET") + price = input_data.get("price", "") + connector = input_data.get("connector_name", "?") + summary = f"{side} {amount} {pair} ({order_type})" + if price: + summary += f" @ {price}" + summary += f" on {connector}" + return summary + + if tool_name == "manage_executors": + action = input_data.get("action", "?") + exec_type = input_data.get("executor_type", "") + exec_id = input_data.get("executor_id", "") + if action == "create" and exec_type: + config = input_data.get("executor_config", {}) + pair = config.get("trading_pair", "?") + return f"Create {exec_type} on {pair}" + if action == "stop" and exec_id: + return f"Stop executor {exec_id[:12]}..." + return f"Executor: {action}" + + if tool_name == "manage_bots": + action = input_data.get("action", "?") + bot_name = input_data.get("bot_name", "?") + if action == "deploy": + controllers = input_data.get("controllers_config", []) + return f"Deploy bot '{bot_name}' with controllers {controllers}" + if action == "update_config": + config_name = input_data.get("config_name", "?") + return f"Update config '{config_name}' on bot '{bot_name}'" + return f"Bot '{bot_name}': {action}" + + if tool_name == "manage_gateway_swaps": + action = input_data.get("action", "?") + pair = input_data.get("trading_pair", "?") + side = input_data.get("side", "?") + amount = input_data.get("amount", "?") + return f"Swap {side} {amount} {pair}" + + if tool_name == "manage_gateway_clmm": + action = input_data.get("action", "?") + if action == "open_position": + return "Open LP position" + if action == "close_position": + return "Close LP position" + return f"CLMM: {action}" + + # Generic fallback + return tool_name diff --git a/condor/runtime/llm_client.py b/condor/runtime/llm_client.py index dae5a9925..22f285cbc 100644 --- a/condor/runtime/llm_client.py +++ b/condor/runtime/llm_client.py @@ -27,6 +27,7 @@ from condor.acp import client as acp_client from condor.acp import pydantic_ai_client as pydantic_ai from condor.preferences import resolve_custom_endpoint +from condor.runtime.toolsets import get_project_dir def build_llm_client( @@ -89,10 +90,7 @@ def build_llm_client( # ACP subprocess models: claude-code, gemini, codex. A Claude model can be # pinned via a suffix ("claude-acp:opus"); ACPClient selects it over the # protocol (session/set_model) after handshake — the bridge ignores - # ANTHROPIC_MODEL. Lazy import: handlers/ must not load at runtime-import - # time (reload_handlers would re-execute into this graph). - from handlers.agents._shared import get_project_dir - + # ANTHROPIC_MODEL. command, model_env, model_pref = acp_client.resolve_acp(agent_key) merged_env = {**(extra_env or {}), **model_env} return acp_client.ACPClient( diff --git a/condor/runtime/sessions.py b/condor/runtime/sessions.py index cc9b9851f..78d52b778 100644 --- a/condor/runtime/sessions.py +++ b/condor/runtime/sessions.py @@ -22,10 +22,10 @@ from condor.agents.agent import identity_header as agent_identity_header from condor.runtime import binding, conversations from condor.runtime.confirmations import get_registry as get_confirmation_registry +from condor.runtime.context import build_initial_context, platform_formatting from condor.runtime.keys import SessionKey from condor.runtime.models import SessionInfo, SessionSpec from condor.runtime.timeouts import TIMEOUTS -from handlers.agents._shared import build_initial_context, platform_formatting log = logging.getLogger(__name__) diff --git a/condor/runtime/toolsets.py b/condor/runtime/toolsets.py new file mode 100644 index 000000000..f52a9a44d --- /dev/null +++ b/condor/runtime/toolsets.py @@ -0,0 +1,240 @@ +"""MCP toolset builders for agent sessions (ARCH-190). + +Builds the per-session MCP server configs (condor + hummingbot subprocesses) +that every surface — Telegram, web dashboard, consult/delegate, the tick +engine — hands to its client. Platform-neutral runtime foundation: this used +to live in ``handlers/agents/_shared.py``, which the runtime could only reach +via lazy imports; ``handlers`` now re-exports from here instead. +""" + +from __future__ import annotations + +import logging +import os +from pathlib import Path +from typing import Any + +from condor.acp.client import bot_process_marker + +log = logging.getLogger(__name__) + + +def get_project_dir() -> str: + """Get the condor project root directory (where .mcp.json lives). + + The ACP agent auto-discovers stdio MCP servers from .mcp.json in the cwd, + so we just need to point it at the project root. + """ + return str(Path(__file__).parent.parent.parent) + + +def _bot_token() -> str: + """This bot's Telegram token, read at call time. + + Goes through ``utils.config`` for its ``load_dotenv()`` side effect, so the + marker built here is derived from the same token ``main.py`` hands the + reaper — a mismatch would silently strand orphaned subprocess trees. + """ + import utils.config # noqa: F401 (imported for load_dotenv()) + + return os.environ.get("TELEGRAM_TOKEN", "") + + +def _bot_id_args() -> list[str]: + """``--bot-id `` for an MCP subprocess, or nothing without a token. + + The digest is what the startup reaper (``reap_stale_acp_trees``) seeds on to + find subprocess trees orphaned by a previous crash. It replaced the raw + token, which used to sit here in clear text (SEC-095). Both MCP servers + parse their argv with ``parse_known_args``, so the flag is inert to them — + it exists purely to be visible in ``ps``. + """ + marker = bot_process_marker(_bot_token()) + return ["--bot-id", marker] if marker else [] + + +def _env_entries(**values: Any) -> list[dict[str, str]]: + """ACP ``env`` entries (``{"name", "value"}``), skipping empty values. + + This is the channel every secret an MCP subprocess needs must travel on: the + ACP bridge and the pydantic-ai stdio client both overlay these onto the + child's inherited environment, which — unlike argv — no other local user can + read out of ``ps``. + + Values are coerced to ``str``: YAML loads an unquoted numeric credential as + int (e.g. ``password: 123``), and an int in the environment mapping breaks + subprocess spawning — for pydantic-ai backends (lmstudio:/ollama:/ + openrouter:) it surfaces as a StdioServerParameters validation error. + """ + entries = ( + (name, str(value)) for name, value in values.items() if value is not None + ) + return [{"name": name, "value": value} for name, value in entries if value] + + +def _condor_mcp_args( + chat_id: int | str, + user_id: int, + agent_slug: str | None = None, + server_name: str | None = None, + delegate_worker: bool = False, +) -> list[str]: + """Build CLI args for the condor MCP subprocess. + + ``delegate_worker`` marks a *background Condor worker* — the detached session + ``delegate`` starts (FEAT-032). The chat and that worker share one agent + record, so the flag is what tells the subprocess which seat it is sitting in. + + The bot token travels in the server's ``env``, never here: argv is + world-readable via ``ps`` (SEC-095). What argv carries instead is + ``--bot-id``, the token's non-secret digest, which is how the startup reaper + recognizes our own leaked subprocess trees. + """ + # MCP server expects int chat_id. For web sessions (string keys like "web_42"), + # use user_id instead — in Telegram DMs, chat_id == user_id anyway. + effective_chat_id = chat_id if isinstance(chat_id, int) else user_id + args = [ + "--chat-id", + str(effective_chat_id), + "--user-id", + str(user_id), + ] + args.extend(_bot_id_args()) + if agent_slug: + args.extend(["--agent-slug", str(agent_slug)]) + if server_name: + args.extend(["--server-name", str(server_name)]) + if delegate_worker: + args.append("--delegate-worker") + return args + + +def _hummingbot_mcp_args(server: dict[str, Any], server_name: str) -> list[str]: + """Build CLI args for the hummingbot MCP subprocess. + + Only non-secret coordinates travel here — the API username/password go in + the server's ``env`` instead, since argv is world-readable via ``ps`` + (SEC-095). Every element is a string: YAML loads unquoted numerics as int + (e.g. ``port: 8000``), and pydantic-ai's StdioServerParameters rejects + non-string args when starting LM Studio / other local-model sessions. + """ + api_url = f"http://{server['host']}:{server['port']}" + return [ + "run", + "python", + "-m", + "mcp_servers.hummingbot_api", + "--url", + api_url, + "--server-name", + str(server_name), + ] + _bot_id_args() + + +def build_mcp_servers_for_session( + user_id: int, + chat_id: int | str, + user_data: dict | None = None, + server_name: str | None = None, + agent_slug: str | None = None, + delegate_worker: bool = False, +) -> list[dict[str, Any]]: + """Build dynamic MCP server configs for an agent session. + + Returns ACP-format mcpServers that override the static .mcp.json entries by + name. Always includes the condor MCP server; hummingbot is added when a + valid server can be resolved for the user. + + ``server_name`` pins the run to that Condor server (an Agent with + ``server_required``); when omitted, the chat's ambient server is resolved + from the user's preferences, then from the first accessible server. Each + candidate must exist *and* be reachable by ``user_id`` — a name this user + has no grant on resolves to the next candidate, never to its credentials. + + ``agent_slug`` scopes the condor MCP tools' memory/skills to that Agent's + own stores (``agents/{slug}/``). Without it the tools target the chat + condor's stores — correct for chat sessions, wrong for an Agent run: a + serverless consult/tick would silently read and write the CHAT's memory + and skills instead of the Agent's own (e.g. an agent unable to find its + inherited ``routine_cookbook`` skill). + + ``delegate_worker`` marks a background Condor delegation (FEAT-032) so the + subprocess picks up the worker framing and refuses to delegate again. + """ + from config_manager import get_config_manager, get_effective_server + + cm = get_config_manager() + + # Resolve which hummingbot server to use (explicit override > user + # preferences). Every candidate is held to existence *and* reach, because + # the name that comes out of here decides whose API credentials go into the + # subprocess env below. ``chat_id`` is not a principal — ``chat_defaults`` + # is a global map keyed by chat, so a caller who can name someone else's + # chat could name their server too (SEC-178, the SEC-164 shape). The subject + # is always ``user_id``, the authenticated owner of the run; the same + # predicate guards TickEngine._resolve_server. + def usable(name: str | None) -> bool: + return bool(name and cm.get_server(name)) and cm.has_server_access( + user_id, name + ) + + def candidates(): + # Lazy on purpose: resolving the chat default writes back into + # ``user_data``, so it must not run when a pin already answered. + yield server_name + yield get_effective_server(chat_id, user_data) + accessible = cm.get_accessible_servers(user_id) + yield accessible[0] if accessible else None + + server_name = next((name for name in candidates() if usable(name)), None) + + # Condor MCP -- runs as stdio subprocess, tools work locally without TCP bridge + # Pass resolved server_name so start_agent uses the correct server + condor = { + "name": "condor", + "command": "uv", + "args": ["run", "python", "-m", "mcp_servers.condor"] + + _condor_mcp_args( + chat_id, + user_id, + agent_slug, + server_name=server_name, + delegate_worker=delegate_worker, + ), + "env": _env_entries(TELEGRAM_BOT_TOKEN=_bot_token()), + } + + if not server_name: + log.warning( + "No accessible server for user %s (chat %s) — " + "agent will start without mcp-hummingbot", + user_id, + chat_id, + ) + return [condor] + + server = cm.get_server(server_name) + if not server: + log.warning( + "Server '%s' resolved for user %s but not found in servers config — " + "agent will start without mcp-hummingbot", + server_name, + user_id, + ) + return [condor] + + # Credentials go in env, not argv: the API username/password used to sit on + # the command line, where any local `ps` recovered them (SEC-095). The + # non-secret coordinates (url, server name) stay on argv, where they make a + # running subprocess identifiable. + mcp_hummingbot = { + "name": "mcp-hummingbot", + "command": "uv", + "args": _hummingbot_mcp_args(server, server_name), + "env": _env_entries( + HUMMINGBOT_API_USERNAME=server["username"], + HUMMINGBOT_API_PASSWORD=server["password"], + ), + } + + return [mcp_hummingbot, condor] diff --git a/condor/setup_llm.py b/condor/setup_llm.py index fac60277f..1fa7e000f 100644 --- a/condor/setup_llm.py +++ b/condor/setup_llm.py @@ -8,7 +8,7 @@ prints the same report and changes nothing. The wizard owns none of the detection: it renders -:mod:`handlers.agents.readiness` over :data:`handlers.agents._shared.AGENT_OPTIONS`, +:mod:`condor.llm.readiness` over :data:`condor.llm.options.AGENT_OPTIONS`, so a provider added to either shows up here for free and can never disagree with what the ``get_available_models`` MCP tool reports. @@ -29,14 +29,14 @@ import httpx from condor.acp.pydantic_ai_client import DEFAULT_BASE_URLS -from handlers.agents import readiness -from handlers.agents._shared import ( +from condor.llm import readiness +from condor.llm.openrouter_models import fetch_models +from condor.llm.options import ( AGENT_OPTIONS, RECOMMENDED_AGENT, selectable_agent_options, ) -from handlers.agents.openrouter_models import fetch_models -from handlers.agents.readiness import MISSING, READY, UNVERIFIED, Readiness +from condor.llm.readiness import MISSING, READY, UNVERIFIED, Readiness REPO_ROOT = Path(__file__).resolve().parent.parent ENV_PATH = REPO_ROOT / ".env" diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index 95ea9b911..05c81ec01 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -1189,8 +1189,8 @@ async def update_agent_config( write the MCP ``manage_trading_agent`` tool already performs — the web layer simply had no door to it, which is why the UI could only offer a text editor. """ + from condor.llm.options import AGENT_OPTIONS from config_manager import get_config_manager - from handlers.agents._shared import AGENT_OPTIONS agent = _get_agent(slug) diff --git a/condor/web/routes/bots.py b/condor/web/routes/bots.py index 996f64b52..fe06c2b7e 100644 --- a/condor/web/routes/bots.py +++ b/condor/web/routes/bots.py @@ -8,6 +8,7 @@ import yaml from fastapi import APIRouter, Depends, HTTPException +from condor.controller_configs import clean_config_for_save from condor.fetchers.bots import build_bots_page, extract_bots_list from condor.web.auth import require_server_access from condor.web.models import ( @@ -24,7 +25,6 @@ ) from condor.web.routes._errors import upstream_error from config_manager import get_config_manager -from handlers.bots._shared import clean_config_for_save logger = logging.getLogger(__name__) diff --git a/condor/web/routes/chat_ws.py b/condor/web/routes/chat_ws.py index ecb632e3d..0652c0a1b 100644 --- a/condor/web/routes/chat_ws.py +++ b/condor/web/routes/chat_ws.py @@ -13,6 +13,8 @@ from fastapi import APIRouter, Depends, Query, WebSocket, WebSocketDisconnect +from condor.llm.openrouter_models import fetch_models +from condor.llm.options import DEFAULT_AGENT from condor.runtime import WEB, EventType, PromptRequest, SessionKey, SessionSpec from condor.runtime import client as runtime from condor.runtime import conversations @@ -27,8 +29,6 @@ from condor.runtime.wake import register_note_sink, register_sink_factory from condor.web.auth import decode_jwt, extract_ws_token, get_current_user from condor.web.models import WebUser -from handlers.agents._shared import DEFAULT_AGENT -from handlers.agents.openrouter_models import fetch_models log = logging.getLogger(__name__) diff --git a/condor/web/routes/sessions.py b/condor/web/routes/sessions.py index d2a484605..deba0eac9 100644 --- a/condor/web/routes/sessions.py +++ b/condor/web/routes/sessions.py @@ -85,12 +85,12 @@ async def session_options(user: WebUser = Depends(get_current_user)): is a real key. """ from condor.agents.agent import AgentStore + from condor.llm.options import AGENT_OPTIONS, DEFAULT_AGENT from condor.preferences import ( get_active_agent_key, get_custom_providers, load_user_data_for, ) - from handlers.agents._shared import AGENT_OPTIONS, DEFAULT_AGENT cm = get_config_manager() providers = get_custom_providers(load_user_data_for(user.id)) diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index 746f40040..b2d1709d2 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -800,17 +800,17 @@ async def add_custom_provider( the key is accepted, and the response is OpenAI-shaped — all in one round trip — and returns the model list the caller needs anyway. """ + from condor.llm.custom_models import ( + CustomProviderError, + fetch_models, + normalize_base_url, + ) from condor.preferences import ( load_user_data_for, save_custom_provider, suggest_provider_name, unique_provider_name, ) - from handlers.agents.custom_models import ( - CustomProviderError, - fetch_models, - normalize_base_url, - ) raw_url = (body.get("base_url") or "").strip() api_key = (body.get("api_key") or "").strip() @@ -847,8 +847,8 @@ async def get_custom_provider_models( user: WebUser = Depends(get_current_user), ): """Fetch the current chat model list for a saved endpoint.""" + from condor.llm.custom_models import CustomProviderError, fetch_models from condor.preferences import find_custom_provider, load_user_data_for - from handlers.agents.custom_models import CustomProviderError, fetch_models provider = find_custom_provider(load_user_data_for(user.id), name) if provider is None: diff --git a/handlers/agents/_shared.py b/handlers/agents/_shared.py index d51a29f42..90a38f846 100644 --- a/handlers/agents/_shared.py +++ b/handlers/agents/_shared.py @@ -1,67 +1,55 @@ -"""Constants and MCP config loader for agent sessions.""" +"""Telegram-side agent helpers, plus re-exports of the moved core (ARCH-190). + +What remains here is genuinely Telegram-flavored: the chat↔agent binding and +conversation bookkeeping. The platform-neutral foundations moved down into +``condor/`` — the model catalog (:mod:`condor.llm.options`), the MCP toolset +builders (:mod:`condor.runtime.toolsets`), the danger classification +(:mod:`condor.runtime.danger`) and the context builders +(:mod:`condor.runtime.context`) — and are re-exported below so existing +handler imports keep working. New code should import from the new homes. +""" import hashlib -import json import logging -import os -from pathlib import Path -from typing import Any -from condor.acp.client import bot_process_marker -from condor.memory.paths import CHAT_SLUG +# Re-exports for Telegram callers (ARCH-190) — the implementations live in +# condor/ now; condor/ itself must never import back through this module. +from condor.llm.options import ( # noqa: F401 + AGENT_OPTIONS, + DEFAULT_AGENT, + RECOMMENDED_AGENT, + selectable_agent_options, +) +from condor.runtime.context import ( # noqa: F401 + _TELEGRAM_FORMATTING, + _WEB_FORMATTING, + COMPACT_CONTEXT_TEMPLATE, + COMPACT_PROMPT_AUTO, + COMPACT_PROMPT_CUSTOM_TEMPLATE, + build_agent_context, + build_initial_context, + platform_formatting, +) +from condor.runtime.danger import ( # noqa: F401 + BLOCKED_TOOLS, + DANGEROUS_BOT_ACTIONS, + DANGEROUS_CLMM_ACTIONS, + DANGEROUS_EXECUTOR_ACTIONS, + DANGEROUS_SWAP_ACTIONS, + DANGEROUS_TOOLS, + is_dangerous_tool_call, + tool_call_input, + tool_call_name, +) +from condor.runtime.toolsets import ( # noqa: F401 + _bot_id_args, + build_mcp_servers_for_session, + get_project_dir, +) log = logging.getLogger(__name__) -def _chat_agent(): - """The default agent's record — Condor's ``agents/condor/AGENT.md``. - - There is one loader and one frontmatter schema (FEAT-033): the chat is read - by ``AgentStore`` exactly like a specialist is. Returns ``None`` only if the - file is missing or unreadable, which callers treat as "no instructions" - rather than as a startup failure. - """ - from condor.agents.agent import AgentStore - - return AgentStore().get(CHAT_SLUG) - - -AGENT_OPTIONS: dict[str, dict[str, Any]] = { - "claude-code": {"label": "Claude Code"}, - "claude-acp:opus": {"label": "Claude (ACP) — Opus"}, - "claude-acp:sonnet": {"label": "Claude (ACP) — Sonnet"}, - "gemini": {"label": "Gemini CLI"}, - "copilot": {"label": "GitHub Copilot CLI"}, - "codex": {"label": "ChatGPT Codex"}, - "ollama:": {"label": "Ollama — Default Model"}, - "lmstudio:": {"label": "LM Studio — Default Model"}, - # Sentinels — these open a picker instead of being a selectable model, so - # they are NOT valid agent keys. `picker` marks them for surfaces that - # render AGENT_OPTIONS as a flat list of choices (the web dashboard's model - # dropdown), which would otherwise offer a key that fails at session start. - # - # OpenRouter: stored agent_llm becomes "openrouter:". - "openrouter:": {"label": "OpenRouter — Pick Model", "picker": True}, - # Custom endpoints: the user's saved OpenAI-compatible endpoints live in - # preferences (condor/preferences.py, shared with the web dashboard) and - # the stored agent_llm becomes "custom@:". - "custom:": {"label": "Custom — OpenAI-compatible API", "picker": True}, -} - - -# What a fresh install should pick unless it has a reason not to: the ACP bridge -# is the best-supported path, and Sonnet is the balance of capability and cost -# most users want. Surfaces that rank the catalog (the `make pick-model` wizard) -# float this key to the top and mark it; it is deliberately NOT part of the -# `label`, which is reused as a plain display name all over the bot. -RECOMMENDED_AGENT = "claude-acp:sonnet" - - -def selectable_agent_options() -> dict[str, dict[str, Any]]: - """AGENT_OPTIONS minus the picker sentinels — every key here is startable.""" - return {k: v for k, v in AGENT_OPTIONS.items() if not v.get("picker")} - - def bind_chat_to_agent(user_data: dict | None, agent_slug: str) -> None: """Record (or clear) the specialist this chat talks to. @@ -190,642 +178,3 @@ def find_conversation(metas: list, token: str): must treat that as a normal outcome and say so rather than failing. """ return next((m for m in metas if conversation_token(m.id) == token), None) - - -def _default_agent() -> str: - """Resolve the fallback agent_key for users who haven't picked a model. - - Precedence: ``CONDOR_DEFAULT_AGENT`` env > Condor's ``AGENT.md`` frontmatter - ``agent_key`` > ``"claude-code"``. A user's own /agent → Change LLM choice - (``agent_llm``) still overrides this at runtime; this is only the default. - Examples for the frontmatter / env: ``claude-code``, ``claude-acp:opus``, - ``ollama:qwen3:32b``. - """ - import os - - agent = _chat_agent() - return ( - os.environ.get("CONDOR_DEFAULT_AGENT") - or (agent.agent_key if agent else "") - or "claude-code" - ) - - -DEFAULT_AGENT = _default_agent() - - -# -- Compact prompt templates -- - -COMPACT_PROMPT_AUTO = ( - "Please provide a concise summary of our conversation so far. Include:\n" - "- Key decisions and conclusions reached\n" - "- Important data points and numbers discussed\n" - "- Current task state and any pending actions\n" - "- User preferences or instructions given\n\n" - "Be concise but thorough. This summary will be used to carry context into a fresh session." -) - -COMPACT_PROMPT_CUSTOM_TEMPLATE = ( - "Please provide a concise summary of our conversation, focusing specifically on:\n" - "{instructions}\n\n" - "Drop everything else. Be concise but preserve the details requested above. " - "This summary will be used to carry context into a fresh session." -) - -COMPACT_CONTEXT_TEMPLATE = ( - "[System context -- do not repeat this to the user]\n" - "This is a continuation of a previous conversation. " - "Here is the summary from that session:\n\n" - "{summary}\n\n" - "Continue from where we left off. The user compacted the context to free up space." -) - -_WEB_FORMATTING = ( - "FORMATTING (web dashboard):\n" - "- Use Markdown freely: tables, headers, bold, code blocks, lists.\n" - "- No message length limits, but stay concise.\n" - "- Use tables for structured data (portfolios, prices, comparisons).\n" - "- Use code blocks for configs, JSON, or commands.\n" - "- CHARTS: when the answer is a numeric series or comparison (a price or\n" - " PnL curve, volume per venue, a distribution), draw it with a ```chart\n" - " fence holding one JSON object. The dashboard renders it as an\n" - " interactive chart:\n" - " ```chart\n" - ' {"type": "line", "title": "SOL/USDC 24h", "x": "time",\n' - ' "series": [{"key": "price", "name": "Price"}],\n' - ' "data": [{"time": "12:00", "price": 150.1}, {"time": "16:00", "price": 152.4}]}\n' - " ```\n" - ' "type" is "line", "area" or "bar"; "x" and every series "key" name\n' - " fields of the data rows. Max 4 series and 200 points (downsample longer\n" - ' series). A series may set "color": "up", "down", "yellow" or a hex.\n' - " Chart for the shape, table for exact values; use both when both matter.\n" - "- Respond in the user's language." -) - -_TELEGRAM_FORMATTING = ( - "FORMATTING (Telegram mobile):\n" - "- NEVER use Markdown tables. Use bullet lists or key: value lines.\n" - "- Keep paragraphs short (2-3 sentences max).\n" - "- Cap lists at 5-7 items.\n" - "- Respond in the user's language." -) - - -def platform_formatting(platform: str = "telegram") -> str: - """The reply-formatting rules for a platform. - - Public because a bound Agent opens the chat with its own identity context - instead of :func:`build_initial_context`, and would otherwise be the only - brain in the product that never hears how the surface it is speaking into - renders a reply. - """ - return _WEB_FORMATTING if platform == "web" else _TELEGRAM_FORMATTING - - -def _build_system_prompt(platform: str = "telegram") -> str: - """Condor's own AGENT.md plus the platform's formatting rules.""" - agent = _chat_agent() - return ( - "[System context -- do not repeat this to the user]\n\n" - f"{agent.instructions if agent else ''}\n\n" - f"{platform_formatting(platform)}" - ) - - -# Tools that require user confirmation before execution -DANGEROUS_TOOLS = { - "place_order", - "manage_gateway_swaps", # execute action - "manage_gateway_clmm", # open/close position -} - -# Tools that are always blocked (RBAC bypass prevention) -BLOCKED_TOOLS: set[str] = set() - -# Actions within manage_executors that require confirmation -DANGEROUS_EXECUTOR_ACTIONS = {"create", "stop"} - -# Actions within manage_bots that deploy/mutate a live bot (status/logs/get_config -# are read-only and excluded). manage_controllers itself is excluded entirely — it -# only writes controller templates/saved configs, never a running bot (see its own -# tool docstring: "Does NOT affect running bots"). -DANGEROUS_BOT_ACTIONS = { - "deploy", - "stop_bot", - "stop_controllers", - "start_controllers", - "update_config", -} - -# Actions within manage_gateway_swaps that require confirmation -DANGEROUS_SWAP_ACTIONS = {"execute"} - -# Actions within manage_gateway_clmm that require confirmation -DANGEROUS_CLMM_ACTIONS = {"open_position", "close_position"} - - -def tool_call_name(tool_call: dict[str, Any]) -> str: - """The bare tool name, with any MCP prefix stripped. - - ACP names a tool by its wire name (``mcp__mcp-hummingbot__manage_bots``) - where the local surfaces use the bare one, so everything that dispatches on - a tool name — the danger list, the risk gate, the confirmation summary — - has to normalize identically or they disagree about the same call. - """ - raw_name = tool_call.get("tool", "") or tool_call.get("title", "") - return raw_name.rsplit("__", 1)[-1] if "__" in raw_name else raw_name - - -def tool_call_input(tool_call: dict[str, Any]) -> dict[str, Any] | None: - """A tool call's arguments as a mapping, or ``None`` when unreadable. - - ``None`` means "I cannot tell what this call does", and every caller has to - fail closed on it: an action-gated tool whose action can't be read is - treated as dangerous rather than waved through (SEC-093). Callers must not - reach for other spellings of the arguments — the ACP wire's ``rawInput`` is - translated once, at the boundary in :func:`condor.acp.client.normalize_tool_call`. - - A JSON string is parsed: OpenAI-compatible providers deliver tool arguments - that way. - """ - args = tool_call.get("input") - if isinstance(args, dict): - return args - if isinstance(args, str) and args.strip(): - try: - parsed = json.loads(args) - except json.JSONDecodeError: - return None - return parsed if isinstance(parsed, dict) else None - return None - - -def _has_dangerous_action( - tool_call: dict[str, Any], dangerous_actions: set[str] -) -> bool: - """Whether an action-gated tool call selects one of its dangerous actions. - - Fails closed (SEC-093): unreadable arguments, or a missing/non-string - ``action``, count as dangerous. ``action`` is required by every one of - these tools, so an unreadable one is never a benign read — it is a call we - failed to understand, and those belong in front of a human. - """ - input_data = tool_call_input(tool_call) - if input_data is None: - return True - action = input_data.get("action") - if not isinstance(action, str) or not action: - return True - return action in dangerous_actions - - -def is_dangerous_tool_call(tool_call: dict[str, Any]) -> bool: - """Check if a tool call requires user confirmation.""" - tool_name = tool_call_name(tool_call) - - # Direct dangerous tools - if tool_name in DANGEROUS_TOOLS: - # For manage_gateway_swaps, only "execute" action is dangerous - if tool_name == "manage_gateway_swaps": - return _has_dangerous_action(tool_call, DANGEROUS_SWAP_ACTIONS) - - # For manage_gateway_clmm, only open/close are dangerous - if tool_name == "manage_gateway_clmm": - return _has_dangerous_action(tool_call, DANGEROUS_CLMM_ACTIONS) - - return True - - # manage_executors with create/stop actions - if tool_name == "manage_executors": - return _has_dangerous_action(tool_call, DANGEROUS_EXECUTOR_ACTIONS) - - # manage_bots with deploy/stop/update actions (a bot deploy places real - # capital via a controller, a different path than manage_executors) - if tool_name == "manage_bots": - return _has_dangerous_action(tool_call, DANGEROUS_BOT_ACTIONS) - - return False - - -def get_project_dir() -> str: - """Get the condor project root directory (where .mcp.json lives). - - The ACP agent auto-discovers stdio MCP servers from .mcp.json in the cwd, - so we just need to point it at the project root. - """ - return str(Path(__file__).parent.parent.parent) - - -def _bot_token() -> str: - """This bot's Telegram token, read at call time. - - Goes through ``utils.config`` for its ``load_dotenv()`` side effect, so the - marker built here is derived from the same token ``main.py`` hands the - reaper — a mismatch would silently strand orphaned subprocess trees. - """ - import utils.config # noqa: F401 (imported for load_dotenv()) - - return os.environ.get("TELEGRAM_TOKEN", "") - - -def _bot_id_args() -> list[str]: - """``--bot-id `` for an MCP subprocess, or nothing without a token. - - The digest is what the startup reaper (``reap_stale_acp_trees``) seeds on to - find subprocess trees orphaned by a previous crash. It replaced the raw - token, which used to sit here in clear text (SEC-095). Both MCP servers - parse their argv with ``parse_known_args``, so the flag is inert to them — - it exists purely to be visible in ``ps``. - """ - marker = bot_process_marker(_bot_token()) - return ["--bot-id", marker] if marker else [] - - -def _env_entries(**values: Any) -> list[dict[str, str]]: - """ACP ``env`` entries (``{"name", "value"}``), skipping empty values. - - This is the channel every secret an MCP subprocess needs must travel on: the - ACP bridge and the pydantic-ai stdio client both overlay these onto the - child's inherited environment, which — unlike argv — no other local user can - read out of ``ps``. - - Values are coerced to ``str``: YAML loads an unquoted numeric credential as - int (e.g. ``password: 123``), and an int in the environment mapping breaks - subprocess spawning — for pydantic-ai backends (lmstudio:/ollama:/ - openrouter:) it surfaces as a StdioServerParameters validation error. - """ - entries = ( - (name, str(value)) for name, value in values.items() if value is not None - ) - return [{"name": name, "value": value} for name, value in entries if value] - - -def _condor_mcp_args( - chat_id: int | str, - user_id: int, - agent_slug: str | None = None, - server_name: str | None = None, - delegate_worker: bool = False, -) -> list[str]: - """Build CLI args for the condor MCP subprocess. - - ``delegate_worker`` marks a *background Condor worker* — the detached session - ``delegate`` starts (FEAT-032). The chat and that worker share one agent - record, so the flag is what tells the subprocess which seat it is sitting in. - - The bot token travels in the server's ``env``, never here: argv is - world-readable via ``ps`` (SEC-095). What argv carries instead is - ``--bot-id``, the token's non-secret digest, which is how the startup reaper - recognizes our own leaked subprocess trees. - """ - # MCP server expects int chat_id. For web sessions (string keys like "web_42"), - # use user_id instead — in Telegram DMs, chat_id == user_id anyway. - effective_chat_id = chat_id if isinstance(chat_id, int) else user_id - args = [ - "--chat-id", - str(effective_chat_id), - "--user-id", - str(user_id), - ] - args.extend(_bot_id_args()) - if agent_slug: - args.extend(["--agent-slug", str(agent_slug)]) - if server_name: - args.extend(["--server-name", str(server_name)]) - if delegate_worker: - args.append("--delegate-worker") - return args - - -def _hummingbot_mcp_args(server: dict[str, Any], server_name: str) -> list[str]: - """Build CLI args for the hummingbot MCP subprocess. - - Only non-secret coordinates travel here — the API username/password go in - the server's ``env`` instead, since argv is world-readable via ``ps`` - (SEC-095). Every element is a string: YAML loads unquoted numerics as int - (e.g. ``port: 8000``), and pydantic-ai's StdioServerParameters rejects - non-string args when starting LM Studio / other local-model sessions. - """ - api_url = f"http://{server['host']}:{server['port']}" - return [ - "run", - "python", - "-m", - "mcp_servers.hummingbot_api", - "--url", - api_url, - "--server-name", - str(server_name), - ] + _bot_id_args() - - -def build_mcp_servers_for_session( - user_id: int, - chat_id: int | str, - user_data: dict | None = None, - server_name: str | None = None, - agent_slug: str | None = None, - delegate_worker: bool = False, -) -> list[dict[str, Any]]: - """Build dynamic MCP server configs for an agent session. - - Returns ACP-format mcpServers that override the static .mcp.json entries by - name. Always includes the condor MCP server; hummingbot is added when a - valid server can be resolved for the user. - - ``server_name`` pins the run to that Condor server (an Agent with - ``server_required``); when omitted, the chat's ambient server is resolved - from the user's preferences, then from the first accessible server. Each - candidate must exist *and* be reachable by ``user_id`` — a name this user - has no grant on resolves to the next candidate, never to its credentials. - - ``agent_slug`` scopes the condor MCP tools' memory/skills to that Agent's - own stores (``agents/{slug}/``). Without it the tools target the chat - condor's stores — correct for chat sessions, wrong for an Agent run: a - serverless consult/tick would silently read and write the CHAT's memory - and skills instead of the Agent's own (e.g. an agent unable to find its - inherited ``routine_cookbook`` skill). - - ``delegate_worker`` marks a background Condor delegation (FEAT-032) so the - subprocess picks up the worker framing and refuses to delegate again. - """ - from config_manager import get_config_manager, get_effective_server - - cm = get_config_manager() - - # Resolve which hummingbot server to use (explicit override > user - # preferences). Every candidate is held to existence *and* reach, because - # the name that comes out of here decides whose API credentials go into the - # subprocess env below. ``chat_id`` is not a principal — ``chat_defaults`` - # is a global map keyed by chat, so a caller who can name someone else's - # chat could name their server too (SEC-178, the SEC-164 shape). The subject - # is always ``user_id``, the authenticated owner of the run; the same - # predicate guards TickEngine._resolve_server. - def usable(name: str | None) -> bool: - return bool(name and cm.get_server(name)) and cm.has_server_access( - user_id, name - ) - - def candidates(): - # Lazy on purpose: resolving the chat default writes back into - # ``user_data``, so it must not run when a pin already answered. - yield server_name - yield get_effective_server(chat_id, user_data) - accessible = cm.get_accessible_servers(user_id) - yield accessible[0] if accessible else None - - server_name = next((name for name in candidates() if usable(name)), None) - - # Condor MCP -- runs as stdio subprocess, tools work locally without TCP bridge - # Pass resolved server_name so start_agent uses the correct server - condor = { - "name": "condor", - "command": "uv", - "args": ["run", "python", "-m", "mcp_servers.condor"] - + _condor_mcp_args( - chat_id, - user_id, - agent_slug, - server_name=server_name, - delegate_worker=delegate_worker, - ), - "env": _env_entries(TELEGRAM_BOT_TOKEN=_bot_token()), - } - - if not server_name: - log.warning( - "No accessible server for user %s (chat %s) — " - "agent will start without mcp-hummingbot", - user_id, - chat_id, - ) - return [condor] - - server = cm.get_server(server_name) - if not server: - log.warning( - "Server '%s' resolved for user %s but not found in servers config — " - "agent will start without mcp-hummingbot", - server_name, - user_id, - ) - return [condor] - - # Credentials go in env, not argv: the API username/password used to sit on - # the command line, where any local `ps` recovered them (SEC-095). The - # non-secret coordinates (url, server name) stay on argv, where they make a - # running subprocess identifiable. - mcp_hummingbot = { - "name": "mcp-hummingbot", - "command": "uv", - "args": _hummingbot_mcp_args(server, server_name), - "env": _env_entries( - HUMMINGBOT_API_USERNAME=server["username"], - HUMMINGBOT_API_PASSWORD=server["password"], - ), - } - - return [mcp_hummingbot, condor] - - -def build_initial_context( - user_id: int, - chat_id: int | str, - user_data: dict | None = None, - agent_key: str | None = None, - platform: str = "telegram", - server_name: str | None = None, -) -> str: - """Build an initial context prompt telling the agent about server, permissions, and formatting rules.""" - from condor.acp.pydantic_ai_client import is_pydantic_ai_model - from config_manager import ( - ServerPermission, - get_config_manager, - get_effective_server, - ) - - cm = get_config_manager() - - # System prompt: Condor's own AGENT.md + platform formatting - system_prompt = _build_system_prompt(platform) - sections: list[str] = [system_prompt] - - # Resolve active server (explicit override > user preferences) - active_name = server_name or get_effective_server(chat_id, user_data) - accessible = cm.get_accessible_servers(user_id) - if not active_name: - active_name = accessible[0] if accessible else None - - if active_name: - # Build server list with permissions (no credentials needed — MCP is pre-configured) - server_lines: list[str] = [] - for name in accessible: - server = cm.get_server(name) - if not server: - continue - perm = cm.get_server_permission(user_id, name) - perm_label = perm.value.upper() if perm else "UNKNOWN" - active_tag = " (active)" if name == active_name else "" - server_lines.append(f"- {name}{active_tag} [{perm_label}]") - - # Get active server permission for enforcement - active_perm = cm.get_server_permission(user_id, active_name) - active_perm_label = active_perm.value.upper() if active_perm else "UNKNOWN" - - # For ACP agents (Claude Code): instruct them to preload MCP tools via ToolSearch - # Pydantic-ai agents get tools directly, no preload needed - tool_preload_hint = "" - if agent_key and not is_pydantic_ai_model(agent_key): - mcp_tools = [ - "mcp__mcp-hummingbot__get_market_data", - "mcp__mcp-hummingbot__get_portfolio_overview", - "mcp__mcp-hummingbot__manage_executors", - "mcp__mcp-hummingbot__manage_bots", - "mcp__mcp-hummingbot__manage_controllers", - "mcp__mcp-hummingbot__explore_dex_pools", - "mcp__mcp-hummingbot__explore_geckoterminal", - "mcp__mcp-hummingbot__manage_gateway_swaps", - "mcp__mcp-hummingbot__manage_gateway_config", - "mcp__mcp-hummingbot__manage_gateway_container", - "mcp__mcp-hummingbot__search_history", - "mcp__mcp-hummingbot__set_account_position_mode_and_leverage", - "mcp__condor__manage_routines", - "mcp__condor__manage_servers", - "mcp__condor__get_user_context", - "mcp__condor__manage_trading_agent", - "mcp__condor__trading_agent_journal_read", - "mcp__condor__trading_agent_journal_write", - "mcp__condor__send_notification", - "mcp__condor__manage_memory", - "mcp__condor__manage_skill", - ] - tool_preload_hint = ( - "IMPORTANT: At the very start of the session (before your first response), " - "load ALL MCP tools in a single ToolSearch call:\n" - f'ToolSearch(query="select:{",".join(mcp_tools)}")\n' - "This avoids repeated ToolSearch calls that waste context tokens. " - "Do this silently without telling the user." - ) - - # Build server info section - server_info = [ - f"Active server: {active_name} (permission: {active_perm_label})", - "The MCP server is already connected to this server. Do NOT call configure_server — it is pre-configured.", - "", - ] - - if tool_preload_hint: - server_info.extend([tool_preload_hint, ""]) - - server_info.extend( - [ - "Available servers:", - *server_lines, - "", - "Server switching is not supported mid-session. If the user wants a different server, " - "tell them to start a new session with that server selected.", - "", - "Permission rules:", - f"- Your current permission on '{active_name}' is {active_perm_label}.", - "- OWNER: Full access including trading operations and server management.", - "- TRADER: Can trade, view balances, and manage own settings.", - "- Enforce the permission level for this server.", - ] - ) - - sections.append("\n".join(server_info)) - - # User memory index — what the chat assistant remembers about this user. This - # store is the chat's own (FEAT-003), not shared with the user's trading - # agents. Inject only the index; bodies are read on demand via - # manage_memory(action="read"). Nothing injected for new users. - try: - from condor.memory import MemoryStore - - memory_index = MemoryStore(user_id).list_index() - if memory_index: - sections.append( - "[USER MEMORY — what you remember about this user]\n" - "Consider this before responding. Read a full memory with " - 'manage_memory(action="read", name="..."). When you learn something ' - 'new and stable about the user, save it with manage_memory(action="write", ...).\n\n' - f"{memory_index}" - ) - except Exception: - pass # Memory is advisory — never block session start on it. - - # Skills index — read-only playbooks the chat assistant ships (know-how + - # steps). Skills are general to the assistant, not learned per user; inject - # only the index, bodies are read on demand via manage_skill(action="read"). - # Nothing injected when the assistant ships none. - try: - from condor.memory import SkillStore - - skills_index = SkillStore().list_index() - if skills_index: - sections.append( - "[SKILLS — check here BEFORE handling a known flow with raw tools]\n" - "Read-only playbooks shipped with the assistant. Before using raw " - "tools for a flow below, read its playbook with " - 'manage_skill(action="read", name="...") and follow the steps — ' - "don't re-derive or hand-roll what a playbook already covers.\n\n" - f"{skills_index}" - ) - except Exception: - pass # Skills are advisory — never block session start on them. - - # Agents index — domain Agents condor can consult (FEAT: coordinator model). - # condor delegates domain work via consult(...) instead of holding the domain's - # tools/context itself. EVERY agent is listed: one missing from this index can - # never be routed to. Nothing injected only when none exist. - try: - from condor.agents.agent import AgentStore - - # The coordinator is not among its own consultees (FEAT-033). - agents_index = AgentStore().list_index(exclude={CHAT_SLUG}) - if agents_index: - sections.append( - "[AGENTS — consult these BEFORE doing domain work with raw tools]\n" - "You are a coordinator. If a request falls in an agent's domain " - "below, delegate it with " - 'consult(agent="", task="...", context="...") instead of ' - "driving the domain's raw tools yourself — the agent has the focused " - "tools and domain memory. Relay a concise summary of its answer.\n\n" - f"{agents_index}" - ) - except Exception: - pass # The agents index is advisory — never block session start on it. - - return "\n\n".join(sections) - - -def build_agent_context( - agent: "Agent", # noqa: F821 — condor.agents.agent.Agent - user_id: int, - task: str, - context: str = "", -) -> str: - """Assemble the prompt for an Agent consult. - - Mirrors :func:`build_initial_context` but for a worker run: the Agent's own - instructions become the system prompt, its domain-scoped memory/skills indexes - are injected (keyed by the Agent slug, FEAT-003 — the shared brain), and the - consult task is appended last. Read on demand via manage_memory/manage_skill - inside the run. - - Those indexes come from :func:`~condor.memory.domain_context`, the same - builder ``binding.agent_identity_context`` composes, so this Agent carries - identical instructions about its own memory whether it is consulted or - chatted with (ARCH-099). - """ - from condor.memory import domain_context - - sections: list[str] = [agent.instructions] - sections.extend(domain_context(agent.slug, user_id)) - - consult = f"[CONSULT REQUEST]\n{task}" - if context: - consult += f"\n\n[CONTEXT FROM CONDOR]\n{context}" - sections.append(consult) - - return "\n\n".join(sections) diff --git a/handlers/agents/confirmation.py b/handlers/agents/confirmation.py index 9ef63f4fb..0f30ca3fc 100644 --- a/handlers/agents/confirmation.py +++ b/handlers/agents/confirmation.py @@ -9,13 +9,11 @@ import logging from typing import Any -from telegram import Bot, InlineKeyboardButton, InlineKeyboardMarkup +from telegram import Bot -from condor.runtime.confirmations import ( - CONFIRMATION_TIMEOUT, - PendingConfirmation, - get_registry, -) +from condor.runtime.channels import TelegramChannel # noqa: F401 (re-export) +from condor.runtime.confirmations import CONFIRMATION_TIMEOUT, get_registry +from condor.runtime.danger import format_tool_summary # noqa: F401 (re-export) log = logging.getLogger(__name__) @@ -28,112 +26,11 @@ ] -def format_tool_summary(tool_call: dict[str, Any]) -> str: - """Format a tool call into a human-readable summary for the confirmation message.""" - from handlers.agents._shared import tool_call_input, tool_call_name - - # The bare name, or an ACP call (``mcp__mcp-hummingbot__manage_bots``) would - # match none of the branches below and be approved as an opaque string. - tool_name = tool_call_name(tool_call) or "Unknown" - input_data = tool_call_input(tool_call) - if input_data is None: - # The gate sends unreadable calls here on purpose (SEC-093). Say so, - # rather than rendering a row of "?" that looks like a parsed call. - return f"{tool_name} (arguments could not be read)" - - if tool_name == "place_order": - side = input_data.get("trade_type", "?") - pair = input_data.get("trading_pair", "?") - amount = input_data.get("amount", "?") - order_type = input_data.get("order_type", "MARKET") - price = input_data.get("price", "") - connector = input_data.get("connector_name", "?") - summary = f"{side} {amount} {pair} ({order_type})" - if price: - summary += f" @ {price}" - summary += f" on {connector}" - return summary - - if tool_name == "manage_executors": - action = input_data.get("action", "?") - exec_type = input_data.get("executor_type", "") - exec_id = input_data.get("executor_id", "") - if action == "create" and exec_type: - config = input_data.get("executor_config", {}) - pair = config.get("trading_pair", "?") - return f"Create {exec_type} on {pair}" - if action == "stop" and exec_id: - return f"Stop executor {exec_id[:12]}..." - return f"Executor: {action}" - - if tool_name == "manage_bots": - action = input_data.get("action", "?") - bot_name = input_data.get("bot_name", "?") - if action == "deploy": - controllers = input_data.get("controllers_config", []) - return f"Deploy bot '{bot_name}' with controllers {controllers}" - if action == "update_config": - config_name = input_data.get("config_name", "?") - return f"Update config '{config_name}' on bot '{bot_name}'" - return f"Bot '{bot_name}': {action}" - - if tool_name == "manage_gateway_swaps": - action = input_data.get("action", "?") - pair = input_data.get("trading_pair", "?") - side = input_data.get("side", "?") - amount = input_data.get("amount", "?") - return f"Swap {side} {amount} {pair}" - - if tool_name == "manage_gateway_clmm": - action = input_data.get("action", "?") - if action == "open_position": - return "Open LP position" - if action == "close_position": - return "Close LP position" - return f"CLMM: {action}" - - # Generic fallback - return tool_name - - -# Backwards-compatible alias for the pre-registry name. +# Backwards-compatible alias for the pre-registry name; the implementation +# moved to condor.runtime.danger (ARCH-190). _format_tool_summary = format_tool_summary -class TelegramChannel: - """Renders a pending confirmation as an inline keyboard in a chat.""" - - def __init__(self, bot: Bot, chat_id: int): - self._bot = bot - self._chat_id = chat_id - - async def deliver(self, pending: PendingConfirmation) -> None: - keyboard = InlineKeyboardMarkup( - [ - [ - InlineKeyboardButton( - "Approve", callback_data=f"agent:confirm_trade:{pending.id}" - ), - InlineKeyboardButton( - "Reject", callback_data=f"agent:reject_trade:{pending.id}" - ), - ] - ] - ) - # Naming the asker is not decoration: one chat can talk to several - # agents on several servers, and "Approve this action?" on its own does - # not say whose action, or where it lands. - asked_by = f"\nRequested by {pending.origin}\n" if pending.origin else "" - await self._bot.send_message( - chat_id=self._chat_id, - text=( - f"Trade Confirmation\n{asked_by}\n" - f"{pending.summary}\n\nApprove this action?" - ), - reply_markup=keyboard, - ) - - async def permission_callback( bot: Bot, chat_id: int, diff --git a/handlers/agents/custom_models.py b/handlers/agents/custom_models.py index b8f212d35..bf30899b0 100644 --- a/handlers/agents/custom_models.py +++ b/handlers/agents/custom_models.py @@ -1,344 +1,11 @@ -"""Custom OpenAI-compatible provider model fetcher. +"""Moved to :mod:`condor.llm.custom_models` (ARCH-190). -Hits GET {base_url}/models with an optional Bearer key — the standard model -list endpoint every OpenAI-compatible API exposes (Venice AI, Together, -Fireworks, vLLM, TGI, ...). A successful fetch doubles as validation of both -the URL and the API key during the custom-endpoint setup flow. +This alias keeps ``handlers.agents.custom_models`` imports (and monkeypatches +on the module's attributes) pointing at the one real module object. """ -from __future__ import annotations +import sys -import asyncio -import hashlib -import ipaddress -import logging -import os -import re -import socket -from urllib.parse import urlparse +from condor.llm import custom_models as _impl -import aiohttp - -log = logging.getLogger(__name__) - -# Per-candidate budget. Candidates are probed concurrently, so this is also -# roughly the worst case for the whole fetch — the user is staring at a -# "validating..." message the entire time. -FETCH_TIMEOUT_SECONDS = 8 - -# Hosts that hand out cloud instance credentials to anything that can issue an -# HTTP GET. Always refused: no legitimate model endpoint lives here. -BLOCKED_HOSTNAMES = frozenset( - { - "metadata.google.internal", - "metadata.goog", - "instance-data", - } -) - -# Opt-in for multi-tenant deployments: also refuse loopback and RFC1918 -# targets. Off by default because pointing at a local Ollama/vLLM/LM Studio -# server is the single most common use of this feature. -_BLOCK_PRIVATE_ENV = "CUSTOM_LLM_BLOCK_PRIVATE_URLS" - -# /models entries whose declared type isn't one of these are not chat models. -# Providers disagree on the field name (`type` on Venice/Together, `object` on -# stock OpenAI) and on the vocabulary, so accept every spelling of "text in, -# text out" we've seen and treat other declared values as non-chat. -CHAT_MODEL_TYPES = frozenset({"text", "chat", "language", "llm"}) - -# Values that technically occupy the type field but say nothing about -# modality — stock OpenAI stamps object="model" on embeddings, Whisper and -# DALL·E alike. Treat these as "not declared" and fall through to the id. -UNINFORMATIVE_MODEL_TYPES = frozenset({"model", "list", "object"}) - -# Fallback for providers that return no type at all: drop ids that are -# unmistakably a non-chat modality. -_NON_CHAT_MARKERS = ( - "embed", - "rerank", - "whisper", - "tts", - "text-to-speech", - "speech", - "voice", - "moderation", - "guard", - "upscal", - "stable-diffusion", - "flux", - "dall-e", - "sdxl", - "image", - "vision-encoder", - "clip", - "ocr", -) - - -# urlparse is happy to call "not a url at all" a netloc, so the hostname needs -# its own shape check: dotted labels, or an IPv6 literal in brackets. -_HOSTNAME_RE = re.compile( - r"^(?:[A-Za-z0-9_-]+(?:\.[A-Za-z0-9_-]+)*\.?|\[[0-9A-Fa-f:.]+\])$" -) - - -class CustomProviderError(Exception): - """Raised when the endpoint can't be validated. Message is user-facing.""" - - -def _block_private() -> bool: - return os.environ.get(_BLOCK_PRIVATE_ENV, "").strip().lower() in ( - "1", - "true", - "yes", - "on", - ) - - -def _reject_if_blocked(host: str) -> None: - """Raise ValueError when `host` points somewhere we refuse to fetch from. - - Link-local addresses (the 169.254.169.254 cloud metadata range) are always - refused. Loopback and private ranges are allowed by default so local model - servers work, and can be refused too by setting - CUSTOM_LLM_BLOCK_PRIVATE_URLS=true on shared/multi-tenant deployments. - - Only IP literals are checked unless the strict flag is set — resolving - every hostname would add latency and would still be defeatable by DNS - rebinding, so this is a guard against casual probing, not a security - boundary. - """ - lowered = host.lower().rstrip(".") - if lowered in BLOCKED_HOSTNAMES: - raise ValueError(f"'{host}' is not an allowed endpoint host.") - - strict = _block_private() - addresses: list[str] = [] - try: - ipaddress.ip_address(lowered.strip("[]")) - addresses = [lowered.strip("[]")] - except ValueError: - if strict: - try: - addresses = [info[4][0] for info in socket.getaddrinfo(lowered, None)] - except OSError: - # Unresolvable — let the actual request produce the error - return - - for raw in addresses: - try: - ip = ipaddress.ip_address(raw) - except ValueError: - continue - if ip.is_link_local: - raise ValueError( - f"'{host}' resolves to a link-local address, which is blocked " - "(cloud instance metadata lives there)." - ) - if strict and (ip.is_loopback or ip.is_private or ip.is_reserved): - raise ValueError( - f"'{host}' resolves to a private address and " - f"{_BLOCK_PRIVATE_ENV} is enabled." - ) - - -def normalize_base_url(raw: str) -> str: - """Coerce user input into a usable OpenAI-compatible base URL. - - Adds https:// when no scheme is given and strips trailing slashes and - endpoint suffixes users commonly paste (/chat/completions, /models). - Raises ValueError for input that can't be a URL or that targets a - blocked host. - """ - url = raw.strip() - if not url: - raise ValueError("URL is empty.") - if "://" not in url: - url = f"https://{url}" - url = url.rstrip("/") - - try: - parsed = urlparse(url) - hostname = parsed.hostname - except ValueError: - raise ValueError(f"'{raw.strip()}' is not a valid http(s) URL.") from None - if ( - parsed.scheme not in ("http", "https") - or not parsed.netloc - or not hostname - or not _HOSTNAME_RE.match(hostname) - ): - raise ValueError(f"'{raw.strip()}' is not a valid http(s) URL.") - _reject_if_blocked(hostname) - - for suffix in ("/chat/completions", "/completions", "/models"): - if url.endswith(suffix): - url = url[: -len(suffix)] - break - return url.rstrip("/") - - -def _is_chat_model(entry: dict) -> bool: - """Best-effort guess at whether a /models entry can serve chat completions. - - Providers return embeddings, rerankers, TTS and image models from the same - endpoint. Picking one of those succeeds here and then fails at prompt time - with an opaque upstream error, so filter them out while we still have the - metadata to do it. - """ - declared = entry.get("type") or entry.get("object") - if isinstance(declared, str) and declared.strip(): - normalized = declared.strip().lower() - if normalized not in UNINFORMATIVE_MODEL_TYPES: - return normalized in CHAT_MODEL_TYPES - - capabilities = entry.get("capabilities") - if isinstance(capabilities, dict): - for flag in ("chat", "chat_completions", "supportsChatCompletions"): - if flag in capabilities: - return bool(capabilities[flag]) - - model_id = entry.get("id", "").lower() - return not any(marker in model_id for marker in _NON_CHAT_MARKERS) - - -def _extract_model_ids(payload: object, chat_only: bool = True) -> list[str]: - """Pull model ids from an OpenAI-format /models response. - - Expects {"data": [{"id": "..."}, ...]}. Returns sorted, deduped ids; - empty list when the payload doesn't match the expected shape. With - ``chat_only`` (the default) non-chat modalities are dropped. - """ - if not isinstance(payload, dict): - return [] - data = payload.get("data") - if not isinstance(data, list): - return [] - ids = { - entry["id"].strip() - for entry in data - if isinstance(entry, dict) - and isinstance(entry.get("id"), str) - and entry["id"].strip() - and (not chat_only or _is_chat_model(entry)) - } - return sorted(ids, key=str.lower) - - -async def _probe( - session: aiohttp.ClientSession, candidate: str, headers: dict -) -> tuple[list[str], CustomProviderError | None]: - """Fetch one candidate's model list. Returns (models, error).""" - models_url = f"{candidate}/models" - try: - async with session.get(models_url, headers=headers) as resp: - if resp.status in (401, 403): - return [], CustomProviderError( - f"the provider at {candidate} rejected the API key " - f"(HTTP {resp.status}). Check the key and try again." - ) - if resp.status == 404: - return [], CustomProviderError( - f"no /models endpoint at {candidate} (HTTP 404). " - "Check the base URL — it usually ends in /v1." - ) - if resp.status != 200: - return [], CustomProviderError( - f"the provider at {candidate} returned HTTP {resp.status}." - ) - payload = await resp.json(content_type=None) - except asyncio.CancelledError: - raise - except Exception as e: - return [], CustomProviderError( - f"could not reach {models_url} ({type(e).__name__}). " - "Check the URL and that the server is up." - ) - - model_ids = _extract_model_ids(payload) - if not model_ids: - # An endpoint that answered but listed only embeddings/image models is - # a different failure from one that isn't OpenAI-shaped at all. - if _extract_model_ids(payload, chat_only=False): - return [], CustomProviderError( - f"{models_url} returned models, but none of them are chat " - "models (only embeddings/image/audio)." - ) - return [], CustomProviderError( - f"{models_url} responded but did not return an " - "OpenAI-compatible model list." - ) - return model_ids, None - - -async def fetch_models(base_url: str, api_key: str = "") -> tuple[str, list[str]]: - """Validate an OpenAI-compatible endpoint and return its chat model list. - - Probes {base_url}/models and {base_url}/v1/models concurrently — the second - covers users who pasted the host root (e.g. https://api.venice.ai instead - of https://api.venice.ai/api/v1) — and prefers the first candidate that - works. Returns (resolved_base_url, model_ids). - - Raises CustomProviderError with a user-facing message on any failure — - unreachable host, rejected key, or a non-OpenAI-shaped response. - """ - headers = {"Authorization": f"Bearer {api_key}"} if api_key else {} - candidates = [base_url] - if not base_url.endswith("/v1"): - candidates.append(f"{base_url}/v1") - - timeout = aiohttp.ClientTimeout(total=FETCH_TIMEOUT_SECONDS) - async with aiohttp.ClientSession(timeout=timeout) as session: - results = await asyncio.gather( - *(_probe(session, candidate, headers) for candidate in candidates) - ) - - # Candidate order is preference order: the URL the user actually typed wins - for candidate, (model_ids, _) in zip(candidates, results): - if model_ids: - log.info( - "Custom provider validated: %s (%d chat models)", - candidate, - len(model_ids), - ) - return candidate, model_ids - - for _, error in results: - if error: - raise error - raise CustomProviderError("no reachable endpoint found.") - - -def model_token(model_id: str) -> str: - """Short stable token for a model id, for use in callback_data. - - Telegram caps callback_data at 64 bytes and model ids are unbounded. A - content-derived token also survives the model list being refetched between - render and tap, which a bare list index does not. - """ - return hashlib.blake2s(model_id.encode("utf-8"), digest_size=4).hexdigest() - - -def find_by_token(model_ids: list[str], token: str) -> str | None: - """Resolve a token produced by :func:`model_token` back to its model id.""" - for model_id in model_ids: - if model_token(model_id) == token: - return model_id - return None - - -def format_model_label(model_id: str, limit: int = 38) -> str: - """Short label for inline keyboard buttons. - - Truncates through the middle: for ids like - ``meta-llama/Llama-3.3-70B-Instruct-Turbo-Free`` the tail is what - distinguishes one variant from another, so cutting it off is exactly the - wrong end to lose. - """ - if len(model_id) <= limit: - return model_id - keep = limit - 1 # room for the ellipsis - head = (keep + 1) // 2 - tail = keep - head - return f"{model_id[:head]}…{model_id[-tail:]}" +sys.modules[__name__] = _impl diff --git a/handlers/agents/openrouter_models.py b/handlers/agents/openrouter_models.py index 8a7b17b58..b4ba909a8 100644 --- a/handlers/agents/openrouter_models.py +++ b/handlers/agents/openrouter_models.py @@ -1,119 +1,11 @@ -"""OpenRouter model catalog fetcher. +"""Moved to :mod:`condor.llm.openrouter_models` (ARCH-190). -Hits GET https://openrouter.ai/api/v1/models (public, unauthenticated) and -filters to entries that advertise `tools` in their supported_parameters. -Condor's whole architecture depends on tool-calling, so models without it -are hidden from the picker. - -Results are cached for an hour so paginating the picker doesn't refetch. +This alias keeps ``handlers.agents.openrouter_models`` imports (and +monkeypatches on the module's attributes) pointing at the one real module. """ -from __future__ import annotations - -import logging -import time -from dataclasses import dataclass - -import aiohttp - -log = logging.getLogger(__name__) - -OPENROUTER_MODELS_URL = "https://openrouter.ai/api/v1/models" -CACHE_TTL_SECONDS = 3600 - - -@dataclass(frozen=True) -class OpenRouterModel: - slug: str # e.g. "anthropic/claude-sonnet-4-5" - name: str # human-friendly name from the API - context_length: int # tokens - prompt_price: float # USD per 1M input tokens, 0 if free - completion_price: float - - -_cache: tuple[float, list[OpenRouterModel]] | None = None - - -def _parse_price(value: object) -> float: - """OpenRouter returns prices as strings in USD per token. Convert to per-1M.""" - try: - return float(value) * 1_000_000 # type: ignore[arg-type] - except (TypeError, ValueError): - return 0.0 - - -def _model_supports_tools(entry: dict) -> bool: - params = entry.get("supported_parameters") or [] - return "tools" in params - - -async def fetch_models(force_refresh: bool = False) -> list[OpenRouterModel]: - """Return the list of OpenRouter models that support tool-calling. - - Sorted by provider then model id for predictable pagination. - """ - global _cache - - if not force_refresh and _cache is not None: - cached_at, models = _cache - if time.monotonic() - cached_at < CACHE_TTL_SECONDS: - return models - - timeout = aiohttp.ClientTimeout(total=10) - try: - async with aiohttp.ClientSession(timeout=timeout) as session: - async with session.get(OPENROUTER_MODELS_URL) as resp: - resp.raise_for_status() - payload = await resp.json() - except Exception as e: - log.warning("Failed to fetch OpenRouter models: %s", e) - if _cache is not None: - return _cache[1] # serve stale cache on failure - return [] - - raw = payload.get("data") or [] - models: list[OpenRouterModel] = [] - for entry in raw: - if not isinstance(entry, dict): - continue - slug = entry.get("id") - if not isinstance(slug, str) or not slug or slug.startswith("~"): - continue # skip canonical aliases like "~anthropic/claude-haiku-latest" - if not _model_supports_tools(entry): - continue - pricing = entry.get("pricing") or {} - models.append( - OpenRouterModel( - slug=slug, - name=str(entry.get("name") or slug), - context_length=int(entry.get("context_length") or 0), - prompt_price=_parse_price(pricing.get("prompt")), - completion_price=_parse_price(pricing.get("completion")), - ) - ) - - models.sort(key=lambda m: m.slug.lower()) - _cache = (time.monotonic(), models) - log.info("Fetched %d OpenRouter models with tool support", len(models)) - return models - - -def format_button_label(model: OpenRouterModel) -> str: - """Short label for inline keyboard buttons. Telegram max ~30 chars looks good.""" - label = model.name or model.slug - if len(label) > 38: - label = label[:35] + "..." - return label +import sys +from condor.llm import openrouter_models as _impl -def find_model_by_slug( - models: list[OpenRouterModel], slug: str -) -> OpenRouterModel | None: - """Case-insensitive exact-match lookup by slug.""" - target = slug.strip().lower() - if not target: - return None - for m in models: - if m.slug.lower() == target: - return m - return None +sys.modules[__name__] = _impl diff --git a/handlers/agents/readiness.py b/handlers/agents/readiness.py index 340177322..5852d7cdd 100644 --- a/handlers/agents/readiness.py +++ b/handlers/agents/readiness.py @@ -1,298 +1,11 @@ -"""One answer to "can Condor actually run this model right now?". +"""Moved to :mod:`condor.llm.readiness` (ARCH-190). -Every surface that offers a model — the ``get_available_models`` MCP tool behind -the agent_builder skill, and the setup wizard (``condor/setup_llm.py``) — asks -this module, so they can never disagree about what a machine can run. The probes -source their commands from :data:`condor.acp.ACP_COMMANDS` and their base URLs -from :data:`condor.acp.pydantic_ai_client.DEFAULT_BASE_URLS`, so adding a -provider there lights it up here for free. - -Three states, never two: a CLI bridge that is installed but whose login cannot be -proven is ``UNVERIFIED``, not ready and not missing. The login markers below are -undocumented paths that upstream CLIs may move, so they only ever downgrade a row -from *ready* to *unverified* — never to *missing*, and never enough to refuse a -pick. - -Read-only and side-effect-free. Never returns a key value, and an unreachable -local server is a normal state (the user simply isn't running it), reported as -``MISSING`` with the command that starts it, never raised. +This alias keeps ``handlers.agents.readiness`` imports (and monkeypatches on +the module's attributes) pointing at the one real module object. """ -from __future__ import annotations - -import os -import shutil -import subprocess import sys -from dataclasses import dataclass, field -from pathlib import Path -from typing import Iterable - -import httpx - -from condor.acp import ACP_COMMANDS -from condor.acp.pydantic_ai_client import DEFAULT_BASE_URLS - -READY, UNVERIFIED, MISSING = "ready", "unverified", "missing" - -# Cloud providers Condor can address as ":", mapped to the env -# var holding each key. Presence of the key = the provider is usable. Sourced -# here (not hardcoded per surface) so the set stays in one place. -CLOUD_KEY_ENVS = { - "openrouter": "OPENROUTER_API_KEY", - "openai": "OPENAI_API_KEY", - "anthropic": "ANTHROPIC_API_KEY", - "groq": "GROQ_API_KEY", - "google": "GOOGLE_API_KEY", -} - -# Local, keyless OpenAI-compatible servers. Base URLs come from the resolver's -# own defaults (DEFAULT_BASE_URLS) so they can never drift from what actually -# runs the model at request time. -LOCAL_PREFIXES = ("ollama", "lmstudio") - -# What to run once, interactively, to log a bridge in. Not run from here: the -# wizard reports and instructs, it never drives someone else's OAuth flow. -_LOGIN_COMMANDS = { - "claude-code": "claude", - "claude-acp": "claude", - "gemini": "npx @google/gemini-cli", - "copilot": "npx @github/copilot", - "codex": "codex", -} - -# Bridges launched as a bare command still come from npm; name the package so a -# missing one is actionable instead of just absent. -_BARE_INSTALL = { - "claude-agent-acp": "npm install -g @agentclientprotocol/claude-agent-acp", -} - -_START_COMMANDS = { - "ollama": "start it with `ollama serve`", - "lmstudio": "start LM Studio and enable its local server", -} - - -@dataclass(frozen=True) -class Readiness: - """What a menu row needs: a state, a sentence, and any models behind it.""" - - state: str # READY | UNVERIFIED | MISSING - detail: str # human sentence; when not READY it names the fix - models: list[str] = field(default_factory=list) # local server models; [] elsewhere - - @property - def usable(self) -> bool: - """False only for MISSING — an unproven login must never block a pick.""" - return self.state != MISSING - - -def npx_packages_installed() -> set[str]: - """npm packages resolvable WITHOUT a network install: local, global, npx cache. - - Mirrors how npx resolves a package before falling back to downloading it. - ``shutil.which("npx")`` says nothing about the package — npx exists whenever - Node does — so probing the launcher would mark every ``npx``-run bridge - available on any machine with Node. - """ - roots = [Path.cwd() / "node_modules"] - npm = shutil.which("npm") - if npm: - try: - proc = subprocess.run( - [npm, "root", "-g"], capture_output=True, text=True, timeout=10 - ) - if proc.returncode == 0 and proc.stdout.strip(): - roots.append(Path(proc.stdout.strip())) - except (OSError, subprocess.SubprocessError): - pass # npm unusable = nothing installed via it; local + cache still count - roots.extend((Path.home() / ".npm" / "_npx").glob("*/node_modules")) - - installed: set[str] = set() - for root in roots: - if not root.is_dir(): - continue - for entry in root.iterdir(): - if entry.name.startswith("@"): - installed.update(f"{entry.name}/{sub.name}" for sub in entry.iterdir()) - else: - installed.add(entry.name) - return installed - - -def _keychain_has(service: str) -> bool | None: - """macOS keychain lookup: True found, False absent, None cannot tell.""" - security = shutil.which("security") - if not security: - return None - try: - proc = subprocess.run( - [security, "find-generic-password", "-s", service], - capture_output=True, - text=True, - timeout=5, - ) - except (OSError, subprocess.SubprocessError): - return None - return proc.returncode == 0 - - -def acp_login_state(base: str) -> bool | None: - """Has this bridge's interactive login already happened? - - ``True`` a credential exists, ``False`` none was found where that CLI keeps - one, ``None`` this bridge has no marker we can read (GitHub Copilot) or the - machine could not be asked. Heuristic by construction: a credentials file - proves a login happened, not that the token is still live, and its absence - proves nothing on a machine using another auth path — which is why callers - treat False and None alike as *unverified*, never as *not installed*. - """ - home = Path.home() - if base in ("claude-code", "claude-acp"): - if (home / ".claude" / ".credentials.json").exists(): - return True - if sys.platform == "darwin": # Claude Code stores it in the login keychain - return _keychain_has("Claude Code-credentials") - return False - if base == "gemini": - if (home / ".gemini" / "oauth_creds.json").exists(): - return True - return bool(os.environ.get("GEMINI_API_KEY")) or False - if base == "codex": - return (home / ".codex" / "auth.json").exists() - return None # copilot and anything new: no documented on-disk marker - - -def install_command(cmd: str) -> str: - """How to install the CLI behind an ACP command line.""" - parts = cmd.split() - if len(parts) > 1 and parts[0] == "npx": - return f"npm install -g {parts[1]}" - if parts and parts[0] in _BARE_INSTALL: - return _BARE_INSTALL[parts[0]] - return f"install `{cmd.split()[0] if parts else cmd}` and put it on PATH" - - -def acp_bridges() -> list[dict]: - """ACP CLI bridges: whether each is installed, and whether it is logged in. - - A bridge launched as ``npx …`` is available only if the PACKAGE is - already installed; a bare command is available if it is on PATH. ``available`` - means "launchable" — a bridge additionally needs its own interactive login, - reported separately as ``logged_in`` (``None`` = no marker to read). - """ - npx_packages = npx_packages_installed() - out: list[dict] = [] - for base, cmd in ACP_COMMANDS.items(): - parts = cmd.split() - if not parts: - available = False - elif parts[0] == "npx": - available = parts[1] in npx_packages if len(parts) > 1 else False - else: - available = shutil.which(parts[0]) is not None - out.append( - { - "agent_key": base, - "command": cmd, - "available": available, - "logged_in": acp_login_state(base) if available else None, - } - ) - return out - - -async def local_servers() -> dict: - """Probe local OpenAI-compatible servers and list their loaded models. - - Unreachable servers are reported (``reachable: false``), not raised — the - user just isn't running that server. - """ - result: dict = {} - timeout = httpx.Timeout(connect=1.5, read=3.0, write=1.5, pool=1.5) - async with httpx.AsyncClient(timeout=timeout) as client: - for prefix in LOCAL_PREFIXES: - base = DEFAULT_BASE_URLS.get(prefix) - entry = {"base_url": base, "reachable": False, "models": []} - if base: - try: - resp = await client.get(f"{base.rstrip('/')}/models") - if resp.status_code == 200: - data = resp.json().get("data", []) - entry["reachable"] = True - entry["models"] = sorted( - m["id"] for m in data if isinstance(m, dict) and m.get("id") - ) - except Exception: - pass # not running = normal, leave reachable=False - result[prefix] = entry - return result - - -def _bridge_readiness(entry: dict) -> Readiness: - base, cmd = entry["agent_key"], entry["command"] - if not entry["available"]: - return Readiness(MISSING, f"not installed — {install_command(cmd)}") - login = _LOGIN_COMMANDS.get(base, cmd) - if entry.get("logged_in") is True: - return Readiness(READY, "installed and logged in") - if entry.get("logged_in") is False: - return Readiness(UNVERIFIED, f"installed; run `{login}` once and log in") - return Readiness( - UNVERIFIED, - f"installed; login could not be verified — if your first chat " - f"fails, run `{login}` once and log in", - ) - - -def _local_readiness(prefix: str, entry: dict) -> Readiness: - if not entry["reachable"]: - hint = _START_COMMANDS.get(prefix, "start the server") - return Readiness(MISSING, f"not reachable at {entry['base_url']} — {hint}") - models = list(entry["models"]) - return Readiness(READY, f"{len(models)} model(s) available", models) - - -def _key_readiness(base: str, env: dict[str, str]) -> Readiness: - var = CLOUD_KEY_ENVS[base] - if env.get(var) or os.environ.get(var): - return Readiness(READY, f"{var} is set") - if base == "openrouter": - return Readiness(MISSING, "needs an API key (you can enter one here)") - return Readiness(MISSING, f"needs {var} in .env") - - -async def probe_all( - bases: Iterable[str], env: dict[str, str] | None = None -) -> dict[str, Readiness]: - """Readiness for several provider bases, probing each source exactly once. - - ``env`` is passed in rather than read from the process so the setup wizard - can probe against a ``.env`` the bot has not loaded yet; ``os.environ`` is - still consulted as a fallback. - """ - env = env or {} - wanted = list(dict.fromkeys(bases)) - bridges: dict[str, dict] = {} - if any(b in ACP_COMMANDS for b in wanted): - bridges = {e["agent_key"]: e for e in acp_bridges()} - servers: dict = {} - if any(b in LOCAL_PREFIXES for b in wanted): - servers = await local_servers() - - out: dict[str, Readiness] = {} - for base in wanted: - if base in bridges: - out[base] = _bridge_readiness(bridges[base]) - elif base in servers: - out[base] = _local_readiness(base, servers[base]) - elif base in CLOUD_KEY_ENVS: - out[base] = _key_readiness(base, env) - else: - out[base] = Readiness(UNVERIFIED, "cannot be checked from here") - return out +from condor.llm import readiness as _impl -async def probe(base: str, env: dict[str, str] | None = None) -> Readiness: - """Readiness for a single provider base (see :func:`probe_all`).""" - return (await probe_all([base], env))[base] +sys.modules[__name__] = _impl diff --git a/handlers/bots/_shared.py b/handlers/bots/_shared.py index c21a89985..8c4777279 100644 --- a/handlers/bots/_shared.py +++ b/handlers/bots/_shared.py @@ -11,7 +11,6 @@ """ import logging -import re from typing import Any, Dict, List, Optional, Tuple from condor.cache import DEFAULT_CACHE_TTL @@ -189,44 +188,13 @@ def set_controller_config(context, config: Dict[str, Any]) -> None: context.user_data["controller_config_params"] = config -# Matches stringified enum values like "PositionMode.ONEWAY", "OrderType.LIMIT", -# "TradeType.BUY" or the colon variant "PositionMode:ONEWAY". The class name is -# PascalCase and the member starts with an uppercase letter, which avoids matching -# legitimate values such as trading times ("09:30") or token symbols ("USDC.e"). -_ENUM_STR_RE = re.compile(r"^[A-Z][A-Za-z0-9_]*[.:][A-Z][A-Za-z0-9_]*$") - - -def normalize_enum_value(value: Any) -> Any: - """Strip the enum-class prefix from stringified enum values. - - Controller config templates expose enum defaults as strings like - "PositionMode.ONEWAY". Saving them verbatim breaks downstream controller - validation, which expects the plain member name ("ONEWAY"). This recurses - into dicts and lists so nested config values are normalized too. - """ - if isinstance(value, str) and _ENUM_STR_RE.match(value): - return re.split(r"[.:]", value, maxsplit=1)[1] - if isinstance(value, dict): - return {k: normalize_enum_value(v) for k, v in value.items()} - if isinstance(value, list): - return [normalize_enum_value(v) for v in value] - return value - - -def clean_config_for_save(config: Dict[str, Any]) -> Dict[str, Any]: - """Strip internal fields and normalize stringified enum values before saving. - - The backend adds _config_name to YAML files, but Pydantic models with - extra='forbid' reject it on reload. Strip any underscore-prefixed keys - that aren't part of the controller config schema. - - Enum fields (e.g. position_mode, side, take_profit_order_type) can arrive as - prefixed strings like "PositionMode.ONEWAY"; normalize them to the plain - member name so deploy/backtest validation accepts them. - """ - return { - k: normalize_enum_value(v) for k, v in config.items() if not k.startswith("_") - } +# Moved to condor/controller_configs.py (ARCH-190) so the web dashboard can use +# them without importing the handlers package; re-imported here for Telegram +# callers. +from condor.controller_configs import ( # noqa: E402,F401 + clean_config_for_save, + normalize_enum_value, +) def init_new_controller_config( diff --git a/tests/runtime/test_agent_binding.py b/tests/runtime/test_agent_binding.py index c2fbf47eb..bcec33ed4 100644 --- a/tests/runtime/test_agent_binding.py +++ b/tests/runtime/test_agent_binding.py @@ -101,7 +101,7 @@ def fake_session_servers(user_id, chat_id, user_data=None, **kw): return [{"name": "condor"}] monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", fake_session_servers + "condor.runtime.toolsets.build_mcp_servers_for_session", fake_session_servers ) bound = binding.resolve(_spec(agent_key="claude-code")) @@ -162,7 +162,7 @@ def test_a_missing_default_record_still_starts_the_chat(monkeypatch, tmp_path): """An unreadable agents/condor/AGENT.md degrades; it does not fail closed.""" monkeypatch.setattr(agent_module, "_DATA_ROOT", tmp_path) monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) bound = binding.resolve(_spec(agent_key="claude-code")) @@ -181,7 +181,7 @@ def test_a_named_agent_that_does_not_exist_still_raises(monkeypatch, tmp_path): def test_resolve_agent_sets_memory_scope(agents_root, monkeypatch): """A bound Agent scopes the MCP subprocess to its own store.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [{"name": "condor", "agent_slug": kw.get("agent_slug")}], ) @@ -203,7 +203,7 @@ def test_resolve_agent_sets_memory_scope(agents_root, monkeypatch): def test_agent_key_precedence(agents_root, monkeypatch): """Explicit spec model > Agent's configured model.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) inherited = binding.resolve(_spec(agent_slug="executor_manager")) @@ -228,7 +228,7 @@ def test_unknown_agent_slug(agents_root): def test_tool_allowlist_enforced(agents_root, registry, monkeypatch): """A bound pydantic-ai session is restricted to the Agent's tools.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) info = asyncio.run(runtime.create_session(_spec(agent_slug="executor_manager"))) @@ -246,7 +246,7 @@ def test_tool_allowlist_enforced(agents_root, registry, monkeypatch): def test_bound_session_opens_with_agent_identity(agents_root, registry, monkeypatch): """The Agent's own instructions replace the chat assistant's context.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) asyncio.run( @@ -262,7 +262,7 @@ def test_bound_session_opens_with_agent_identity(agents_root, registry, monkeypa def test_unbound_session_keeps_chat_context(registry, monkeypatch): """Without a binding, the assistant context is untouched.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) asyncio.run( @@ -278,7 +278,7 @@ def test_unbound_session_keeps_chat_context(registry, monkeypatch): def test_switch_destroys_and_respawns(agents_root, registry, monkeypatch): """Rebinding stops the old subprocess exactly once and reuses the key.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) key = SessionKey.telegram(42) @@ -309,7 +309,7 @@ async def scenario(): def test_rebinding_to_same_agent_reuses_session(agents_root, registry, monkeypatch): """An empty agent_key must not look like a model change and respawn.""" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **kw: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **kw: [] ) async def scenario(): diff --git a/tests/runtime/test_busy_policy.py b/tests/runtime/test_busy_policy.py index 294390e0d..a614a3c89 100644 --- a/tests/runtime/test_busy_policy.py +++ b/tests/runtime/test_busy_policy.py @@ -78,7 +78,7 @@ def registry(tmp_path, monkeypatch): monkeypatch.setattr("condor.acp.client.ACPClient", _GatedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) return session_module diff --git a/tests/runtime/test_confirmations.py b/tests/runtime/test_confirmations.py index e40b9dac9..da85efde6 100644 --- a/tests/runtime/test_confirmations.py +++ b/tests/runtime/test_confirmations.py @@ -183,7 +183,7 @@ def test_blocked_tool_refused(registry, monkeypatch): the gate still fires the moment something is added to that set. """ monkeypatch.setattr( - "handlers.agents._shared.BLOCKED_TOOLS", {"place_order"}, raising=False + "condor.runtime.danger.BLOCKED_TOOLS", {"place_order"}, raising=False ) channel = _RecordingChannel() cb = build_permission_callback("tg:1", 1, [channel]) diff --git a/tests/runtime/test_conversations.py b/tests/runtime/test_conversations.py index f5db3379b..f9ac559f0 100644 --- a/tests/runtime/test_conversations.py +++ b/tests/runtime/test_conversations.py @@ -677,7 +677,7 @@ def registry(monkeypatch, conv_root): monkeypatch.setattr("condor.acp.client.ACPClient", _ScriptedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) _ScriptedClient.script = [ TextChunk(text="the answer"), diff --git a/tests/runtime/test_session_api.py b/tests/runtime/test_session_api.py index 2e6ab522f..db34086f5 100644 --- a/tests/runtime/test_session_api.py +++ b/tests/runtime/test_session_api.py @@ -57,7 +57,7 @@ def registry(monkeypatch): # Lives in condor.runtime.binding now; patch it at the source so both the # bound and unbound resolution paths see the stub. monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) return session_module diff --git a/tests/runtime/test_session_creation_lock.py b/tests/runtime/test_session_creation_lock.py index b35b61d15..45e533b8b 100644 --- a/tests/runtime/test_session_creation_lock.py +++ b/tests/runtime/test_session_creation_lock.py @@ -65,7 +65,7 @@ def registry(monkeypatch): monkeypatch.setattr("condor.acp.client.ACPClient", _SlowClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) return session_module diff --git a/tests/runtime/test_session_facade.py b/tests/runtime/test_session_facade.py index 078119352..692b73a94 100644 --- a/tests/runtime/test_session_facade.py +++ b/tests/runtime/test_session_facade.py @@ -52,7 +52,7 @@ def registry(monkeypatch): # Lives in condor.runtime.binding now; patch it at the source so both the # bound and unbound resolution paths see the stub. monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) return session_module diff --git a/tests/runtime/test_wake.py b/tests/runtime/test_wake.py index 53cc69378..68a249330 100644 --- a/tests/runtime/test_wake.py +++ b/tests/runtime/test_wake.py @@ -104,7 +104,7 @@ def registry(tmp_path, monkeypatch): monkeypatch.setattr("condor.acp.client.ACPClient", _EchoClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) # Sink factories and the in-flight counts are process-global, exactly like the # confirmation registry: reset both so tests cannot leak into each other. diff --git a/tests/test_agents.py b/tests/test_agents.py index 85e4984d5..4d61c8485 100644 --- a/tests/test_agents.py +++ b/tests/test_agents.py @@ -548,7 +548,7 @@ def _run_create_session(monkeypatch, *, chat_id, user_id): monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") # Resolved through condor.runtime.binding now, so patch it at the source. monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) _FakeACPClient.last_extra_env = None spec = SessionSpec( diff --git a/tests/test_chat_ws_conversations.py b/tests/test_chat_ws_conversations.py index 755920534..98304fb72 100644 --- a/tests/test_chat_ws_conversations.py +++ b/tests/test_chat_ws_conversations.py @@ -71,7 +71,7 @@ def runtime_env(tmp_path, monkeypatch): monkeypatch.setattr("condor.acp.client.ACPClient", _ScriptedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) return tmp_path diff --git a/tests/test_chat_ws_prewarm.py b/tests/test_chat_ws_prewarm.py index e234d7f0b..75073b7b9 100644 --- a/tests/test_chat_ws_prewarm.py +++ b/tests/test_chat_ws_prewarm.py @@ -76,7 +76,7 @@ def runtime_env(tmp_path, monkeypatch): monkeypatch.setattr(chat_ws, "_pending_spawns", {}) monkeypatch.setattr(chat_ws, "_active_prompt_tasks", {}) monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) return tmp_path diff --git a/tests/test_chat_ws_steering.py b/tests/test_chat_ws_steering.py index aaac5a26f..a8c447837 100644 --- a/tests/test_chat_ws_steering.py +++ b/tests/test_chat_ws_steering.py @@ -90,7 +90,7 @@ def ws_env(tmp_path, monkeypatch): monkeypatch.setattr("condor.acp.client.ACPClient", _GatedClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) monkeypatch.setattr(chat_ws, "_active_prompt_tasks", {}) diff --git a/tests/test_chat_ws_wake.py b/tests/test_chat_ws_wake.py index 211ab4c7f..8bfc2b7f0 100644 --- a/tests/test_chat_ws_wake.py +++ b/tests/test_chat_ws_wake.py @@ -64,7 +64,7 @@ def ws_env(tmp_path, monkeypatch): monkeypatch.setattr("condor.acp.client.ACPClient", _EchoClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) monkeypatch.setattr(chat_ws, "_active_prompt_tasks", {}) diff --git a/tests/test_delegate_worker.py b/tests/test_delegate_worker.py index f510030e7..5ed5e333b 100644 --- a/tests/test_delegate_worker.py +++ b/tests/test_delegate_worker.py @@ -305,10 +305,10 @@ async def prompt(self, text): return "done" monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", fake_build + "condor.runtime.toolsets.build_mcp_servers_for_session", fake_build ) monkeypatch.setattr( - "handlers.agents._shared.build_agent_context", lambda *a, **k: "prompt" + "condor.runtime.context.build_agent_context", lambda *a, **k: "prompt" ) monkeypatch.setattr(acp_client_module, "ACPClient", _FakeClient) diff --git a/tests/test_session_model_switch.py b/tests/test_session_model_switch.py index e4b4a0f31..a9cc0c5fe 100644 --- a/tests/test_session_model_switch.py +++ b/tests/test_session_model_switch.py @@ -101,7 +101,7 @@ def env(tmp_path, monkeypatch): monkeypatch.setattr("condor.acp.client.ACPClient", _Client) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr( "condor.runtime.binding.agent_identity_context", lambda *a, **k: "" @@ -112,7 +112,7 @@ def env(tmp_path, monkeypatch): ) monkeypatch.setattr(session_routes, "get_config_manager", FakeConfigManager) monkeypatch.setattr("condor.web.auth.get_config_manager", FakeConfigManager) - monkeypatch.setattr("handlers.agents._shared.DEFAULT_AGENT", FALLBACK) + monkeypatch.setattr("condor.llm.options.DEFAULT_AGENT", FALLBACK) monkeypatch.setattr("condor.web.routes.chat_ws.DEFAULT_AGENT", FALLBACK) AgentStore().create( diff --git a/tests/test_session_server_access.py b/tests/test_session_server_access.py index 9655cf4c9..49392307c 100644 --- a/tests/test_session_server_access.py +++ b/tests/test_session_server_access.py @@ -224,7 +224,7 @@ def registry(monkeypatch): monkeypatch.setattr("condor.acp.pydantic_ai_client.PydanticAIClient", _FakeClient) monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) return session_module diff --git a/tests/test_session_server_switch.py b/tests/test_session_server_switch.py index cc4cc97ea..b1bcee360 100644 --- a/tests/test_session_server_switch.py +++ b/tests/test_session_server_switch.py @@ -90,7 +90,7 @@ def env(tmp_path, monkeypatch): "condor.web.auth.get_config_manager", lambda: FakeConfigManager() ) monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) diff --git a/tests/test_session_switch.py b/tests/test_session_switch.py index 7ae9b75b3..61edb3251 100644 --- a/tests/test_session_switch.py +++ b/tests/test_session_switch.py @@ -72,7 +72,7 @@ def env(tmp_path, monkeypatch): monkeypatch.setattr(session_module, "build_initial_context", lambda *a, **k: "") monkeypatch.setattr(routes, "get_config_manager", lambda: FakeConfigManager()) monkeypatch.setattr( - "handlers.agents._shared.build_mcp_servers_for_session", lambda *a, **k: [] + "condor.runtime.toolsets.build_mcp_servers_for_session", lambda *a, **k: [] ) monkeypatch.setattr("condor.preferences.load_user_data_for", lambda *a, **k: {}) From 93360a1c544293325221a1a066112bd3c6be4a5d Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 21:01:16 +0300 Subject: [PATCH 43/57] (refactor) extract session-ownership PnL attribution engine into condor/agents/attribution The ownership-window tiling that attributes each bot's realized PnL/volume/trades/fees across the sessions that operated it lived inside web/routes/agents.py, hand-synchronized with a parallel merge in agents/performance.py. Both now consume one implementation: condor.agents.attribution owns session ownership resolution, owner-window tiling, the slice-and-merge fold (fees_known heuristic) and the cum_fees_quote fallback, plus the current-owner rule for the live open book. Routes keep only HTTP concerns and the TTL caches. A parity test pins that the web rollup and fetch_agent_performance report identical figures for a session that adopted a long-running bot. (ARCH-191) --- condor/agents/attribution.py | 292 ++++++++++++++++++++++ condor/agents/performance.py | 48 ++-- condor/web/routes/agents.py | 247 ++---------------- tests/test_bot_mode_session_resolution.py | 16 +- tests/test_session_pnl_attribution.py | 116 +++++++-- 5 files changed, 423 insertions(+), 296 deletions(-) create mode 100644 condor/agents/attribution.py diff --git a/condor/agents/attribution.py b/condor/agents/attribution.py new file mode 100644 index 000000000..bc97a78c2 --- /dev/null +++ b/condor/agents/attribution.py @@ -0,0 +1,292 @@ +"""Session-ownership PnL attribution engine ([[FEAT-018]], [[ARCH-191]]). + +Single source of truth for how a bot's realized PnL / volume / trades / fees are +attributed to the sessions that operated it. The vocabulary lives here in one +place: + +- **session ownership resolution** — which bases a session owned and since when, + from the ledger ``owned_bots.json`` writes (:mod:`condor.agents.ownership`) + with a legacy shim for sessions predating it; +- **owner-window tiling** — the per-base ``[since_i, since_{i+1})`` windows that + reproduce a bot's whole cumulative with no gap and no double count; +- **slice-and-merge** — folding a sliced history window into a performance + object, including the shared fee rules (``fees_known`` heuristic and the + live ``cum_fees_quote`` fallback), which exist exactly once, here; +- **the current-owner rule** — live unrealized PnL and the open book belong to + whoever operates the bot now. + +Both consumers go through this module: the web strategy rollup +(``condor.web.routes.agents``) via :func:`apply_bot_mode_pnl` / +:func:`current_owner_bases` / :func:`session_ownership`, and the agent's own +view (``condor.agents.performance``) via :func:`fold_sliced_window` / +:func:`sliced_or_live_fees`. That is what makes the invariant structural +instead of hand-maintained: the dashboard and the tick loop cannot disagree, +because they no longer have separate copies of the rules to drift apart. +""" + +from __future__ import annotations + +import logging +import os +import time +from pathlib import Path +from typing import Any + +from condor.agents.ownership import OwnedBot, read_owned +from condor.agents.sessions_index import find_session_dir + +log = logging.getLogger(__name__) + + +# ── Shared merge rules ── +# ``perf`` is duck-typed over the attribute vocabulary the two consumers share: +# the agents-side ``AgentPerformance`` dataclass and the web's +# ``AgentPerformanceModel`` both carry realized_pnl / unrealized_pnl / total_pnl / +# volume / fees / fees_known / trade_count / closed_count. + + +def fold_sliced_window(perf: Any, window: tuple[float, float, float, float]) -> None: + """Fold one sliced ``(realized, volume, trades, fees)`` window into ``perf``. + + Sliced closes are round-trip closes, so they are this window's trades AND + its closed positions — the same two counters bumped on both surfaces, so the + session detail and the rollup report one number. + """ + realized, volume, trades, fees = window + closes = int(round(trades)) + perf.realized_pnl += realized + perf.volume += volume + perf.fees += fees + # Traded but charged nothing: the fee column is missing, not the fees. + if volume > 0 and fees == 0.0: + perf.fees_known = False + perf.trade_count += closes + perf.closed_count += closes + perf.total_pnl = perf.realized_pnl + perf.unrealized_pnl + + +def sliced_or_live_fees(sliced_fees: float, bot: dict[str, Any]) -> float: + """Fees from the sliced history, else the live snapshot's cumulative figure. + + A backend that reports no cumulative fee column slices to zero; the live + open-position figure is then the only one there is — attributed rather than + silently dropped. + """ + if sliced_fees: + return sliced_fees + return float(bot.get("cum_fees_quote", 0) or 0) + + +# ── Session ownership resolution ── + + +def session_bot_base(strategy_dir: Path, default_config: dict | None, num: int) -> str: + """Bot base name a session operates: per-session config, else strategy default. + + A non-empty per-session ``bot_name`` wins (so runtime-named bots that record + their deployed name resolve), but an empty/absent one falls back to the + strategy default — early sessions predating the config's ``bot_name`` saved it + as ``''`` and must still map to the shared bot they operated. Empty string when + neither is set (direct-executor strategies). Shared by the per-session PnL + distribution and the operator's live-executor view so both resolve identically. + """ + from condor.agents.config import load_full_config + + default_base = (default_config or {}).get("bot_name", "") or "" + sd = find_session_dir(strategy_dir, num) + if not sd: + return default_base + return load_full_config(sd, default_config).get("bot_name", "") or default_base + + +def session_start_epoch(strategy_dir: Path, num: int) -> float: + """Session start time: config.yml is written once at start, so its mtime is stable.""" + sd = find_session_dir(strategy_dir, num) + if not sd: + return 0.0 + cfg = sd / "config.yml" + target = cfg if cfg.exists() else sd + try: + return os.path.getmtime(target) + except OSError: + return 0.0 + + +def session_ownership( + strategy_dir: Path, default_config: dict | None, num: int +) -> list[OwnedBot]: + """Bases a session owned and the instant it took each over, oldest first. + + Two sources, in order: + + 1. ``{session_dir}/owned_bots.json`` — the ledger [[FEAT-017]] writes, which + knows both the bases (a session may operate several) and the exact takeover + instant, whether the bot was deployed here or adopted after a restart. + 2. the legacy shim — a session predating the ledger resolves its single + ``bot_name`` as one owned bot ``since`` the session started, reproducing the + session-start tiling attribution used before the ledger existed. + + Empty for direct-executor strategies, whose per-session executor attribution + already stands and must not be touched. + """ + owned = read_owned(find_session_dir(strategy_dir, num)) + if owned: + return owned + base = session_bot_base(strategy_dir, default_config, num) + if not base: + return [] + start = session_start_epoch(strategy_dir, num) + return [OwnedBot(base=base, origin="legacy", since=start, last_seen=start)] + + +# ── Owner-window tiling ── + + +def owner_windows( + real_sessions: list, strategy_dir: Path, default_config: dict | None +) -> dict[str, list[tuple[float, Any, float]]]: + """``{base: [(since, session, until), …]}`` — owners, oldest takeover first. + + The windows a base's owners occupy tile ``[since_i, since_{i+1})`` and the last + one runs to now, so slicing over them reproduces the bot's whole cumulative with + no gap and no double count. Keyed per base rather than globally per session + number: two bases handed over at different moments never share a timeline. + + ``until`` is the instant the session released the bot, or ``0.0`` while it + still holds it. A released last window stops there rather than running to now, + which is the one case where the tiling deliberately leaves a gap: PnL a bot + earned with no session operating it belongs to no session. + """ + owners: dict[str, list[tuple[float, Any, float]]] = {} + for s in real_sessions: + for ob in session_ownership(strategy_dir, default_config, s.session_num): + owners.setdefault(ob.base, []).append((ob.since, s, ob.until)) + for lst in owners.values(): + lst.sort(key=lambda t: (t[0], t[1].session_num)) + return owners + + +def current_owner_bases( + strategy_dir: Path, + default_config: dict | None, + session_nums: list[int], + num: int, +) -> list[str]: + """Bases ``num`` is the CURRENT owner of — the last takeover by ``since``. + + A bot's live open positions belong to whoever operates it now, so this is the + gate for merging them into one session's view. Same rule + :func:`apply_bot_mode_pnl` applies to live unrealized PnL, kept here as one + lookup over the same windows so the rollup and the per-session detail can + never disagree about who holds the open book. A session that released the bot + is not its current owner, so an ended session shows no live open book. + """ + last: dict[str, tuple[float, int, float]] = {} + for n in session_nums: + for ob in session_ownership(strategy_dir, default_config, n): + if last.get(ob.base, (float("-inf"), -1, 0.0))[:2] <= (ob.since, n): + last[ob.base] = (ob.since, n, ob.until) + return sorted( + base for base, (_, owner, until) in last.items() if owner == num and until <= 0 + ) + + +# ── The attribution engine ── + + +async def apply_bot_mode_pnl( + real_sessions: list, strategy_dir: Path, default_config: dict | None, client: Any +) -> None: + """Distribute each owned bot's PnL across the sessions that operated it. + + One rule covers deploy and handover: every owned bot is attributed by slicing + its history over ``[since, next_owner.since or now)``, where ``since`` is the + takeover instant the ownership ledger recorded. A bot the session *deployed* + has no history before its ``since``, so the general rule already hands it the + whole instance — the exact case falls out instead of needing its own branch. + + Live unrealized PnL, fees and open positions go to each base's LAST owner by + ``since`` — a lookup in the ledger where it used to be a ``max(session_num)`` + guess, so a new session that never adopted the bot no longer inherits its + open book. + + Works uniformly for single- and multi-controller bots (history sums controllers + per instance) and for a base re-launched under several instances. Strategies + whose sessions own no bot (direct-executor agents) are left untouched. + """ + from condor.fetchers.bot_performance import ( + bot_executor_rows, + fetch_all_bot_performance, + fetch_archived_instances, + fetch_base_histories, + resolve_bots, + slice_history, + ) + + if not client or not real_sessions: + return + owners = owner_windows(real_sessions, strategy_dir, default_config) + bases = sorted(owners) + if not bases: + return # direct-executor strategy — nothing to attribute + + try: + all_perf = await fetch_all_bot_performance(client) + except Exception as e: + log.warning("bot perf fetch for %s failed: %s", strategy_dir.name, e) + return + + now = time.time() + # The oldest takeover across every base sets how far back the histories must + # reach; sampling resolution is chosen from it so no owner's window falls off + # the end of the retained rows. + earliest = min( + (since for lst in owners.values() for since, _, _ in lst if since > 0), + default=0.0, + ) + # Archived instances carry the realized PnL of every bot a session stopped — + # the normal end state of a finished session, and invisible in the live + # snapshot. Same universe the live agent's own view uses, so the dashboard and + # the tick loop cannot disagree about what a session earned. + archived = await fetch_archived_instances(client) + histories_by_base = await fetch_base_histories( + client, all_perf, bases, earliest, now, extra_names=archived + ) + + live = resolve_bots(all_perf, bases) + for base in bases: + window_owners = owners[base] + insts = histories_by_base.get(base, []) + + # Realized / volume / trades / fees: one window per owner, tiling the + # timeline. A released window (the session stopped and left the bot + # running) ends at its release instant, so PnL earned while no session + # was operating the bot is attributed to nobody instead of accruing to + # whoever happened to hold last. + sliced_fees = 0.0 + for i, (since, s, until) in enumerate(window_owners): + end = window_owners[i + 1][0] if i + 1 < len(window_owners) else now + if until > 0: + end = min(end, until) + if end <= since: + continue + window = slice_history(insts, since, end) + fold_sliced_window(s, window) + sliced_fees += window[3] + + # Live unrealized + open positions → the base's current owner, unless it + # has released the bot: an ended session holds no open book. + bot = live.get(base) + if not bot: + continue + last_since, operator, last_until = window_owners[-1] + if last_until > 0: + continue + b_rows = bot_executor_rows(bot) + operator.unrealized_pnl += float(bot.get("unrealized_pnl_quote", 0) or 0) + # Top the operator up to the fallback figure: when the whole sliced fee + # column is zero, the live open-position fees are the only ones there are. + operator.fees += sliced_or_live_fees(sliced_fees, bot) - sliced_fees + operator.open_count += sum(1 for r in b_rows if r["status"] == "RUNNING") + operator.executors = list(operator.executors) + b_rows + operator.total_pnl = operator.realized_pnl + operator.unrealized_pnl diff --git a/condor/agents/performance.py b/condor/agents/performance.py index 7df8ff81c..149415f5c 100644 --- a/condor/agents/performance.py +++ b/condor/agents/performance.py @@ -167,40 +167,35 @@ def _merge_bot_perf( the bot snapshot reports how many positions closed but not how each one ended, so a bot-mode agent has no per-trade outcome to derive one from. """ + from condor.agents.attribution import fold_sliced_window, sliced_or_live_fees from condor.fetchers.bot_performance import bot_executor_rows rows = bot_executor_rows(bot) open_rows = [r for r in rows if r["status"] == "RUNNING"] if window is None: - closes = int(bot.get("closed_trades", 0) or 0) - volume_added = float(bot.get("volume_traded", 0) or 0) - fees_added = float(bot.get("cum_fees_quote", 0) or 0) - perf.realized_pnl += float(bot.get("realized_pnl_quote", 0) or 0) - perf.volume += volume_added - perf.fees += fees_added + # Lifetime aggregate, folded through the same shared rule as a window so + # the fees_known heuristic exists exactly once (condor.agents.attribution). + fold_sliced_window( + perf, + ( + float(bot.get("realized_pnl_quote", 0) or 0), + float(bot.get("volume_traded", 0) or 0), + float(bot.get("closed_trades", 0) or 0), + float(bot.get("cum_fees_quote", 0) or 0), + ), + ) else: realized, volume, trades, fees = window - closes = int(round(trades)) - perf.realized_pnl += realized - perf.volume += volume - # A backend that reports no cumulative fee column slices to zero; the - # live open-position figure is then the only one there is. - fees_added = fees if fees else float(bot.get("cum_fees_quote", 0) or 0) - volume_added = volume - perf.fees += fees_added - - # Traded but charged nothing: the fee column is missing, not the fees. - if volume_added > 0 and fees_added == 0.0: - perf.fees_known = False + fold_sliced_window( + perf, (realized, volume, trades, sliced_or_live_fees(fees, bot)) + ) for ct, n in (bot.get("close_type_counts") or {}).items(): perf.close_type_counts[str(ct)] = perf.close_type_counts.get(str(ct), 0) + int( n or 0 ) - perf.trade_count += closes - perf.closed_count += closes perf.unrealized_pnl += float(bot.get("unrealized_pnl_quote", 0) or 0) perf.total_pnl = perf.realized_pnl + perf.unrealized_pnl perf.controllers = perf.controllers + list(bot.get("controllers", [])) @@ -297,16 +292,9 @@ def _merge_stopped_bot_perf( the normal way a session ends. Everything an open position would contribute (unrealized PnL, executor rows) is correctly absent. """ - realized, volume, trades, fees = window - closes = int(round(trades)) - perf.realized_pnl += realized - perf.volume += volume - perf.fees += fees - if volume > 0 and fees == 0.0: - perf.fees_known = False - perf.trade_count += closes - perf.closed_count += closes - perf.total_pnl = perf.realized_pnl + perf.unrealized_pnl + from condor.agents.attribution import fold_sliced_window + + fold_sliced_window(perf, window) def _build_perf_from_rows( diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index 05c81ec01..a6faa02c1 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -19,7 +19,6 @@ import asyncio import logging -import os import time from collections import OrderedDict from pathlib import Path @@ -28,7 +27,11 @@ from fastapi import APIRouter, Depends, HTTPException from pydantic import BaseModel, Field -from condor.agents.ownership import OwnedBot +from condor.agents.attribution import ( + apply_bot_mode_pnl, + current_owner_bases, + session_ownership, +) from condor.agents.sessions_index import ( count_experiments, count_sessions, @@ -432,230 +435,12 @@ async def _get_client_for_strategy(strategy_dir: Path, default_config: dict | No return client, server_name -def _session_bot_base(strategy_dir: Path, default_config: dict | None, num: int) -> str: - """Bot base name a session operates: per-session config, else strategy default. - - A non-empty per-session ``bot_name`` wins (so runtime-named bots that record - their deployed name resolve), but an empty/absent one falls back to the - strategy default — early sessions predating the config's ``bot_name`` saved it - as ``''`` and must still map to the shared bot they operated. Empty string when - neither is set (direct-executor strategies). Shared by the per-session PnL - distribution and the operator's live-executor view so both resolve identically. - """ - from condor.agents.config import load_full_config - from condor.agents.sessions_index import find_session_dir - - default_base = (default_config or {}).get("bot_name", "") or "" - sd = find_session_dir(strategy_dir, num) - if not sd: - return default_base - return load_full_config(sd, default_config).get("bot_name", "") or default_base - - -def _session_start_epoch(strategy_dir: Path, num: int) -> float: - """Session start time: config.yml is written once at start, so its mtime is stable.""" - from condor.agents.sessions_index import find_session_dir - - sd = find_session_dir(strategy_dir, num) - if not sd: - return 0.0 - cfg = sd / "config.yml" - target = cfg if cfg.exists() else sd - try: - return os.path.getmtime(target) - except OSError: - return 0.0 - - -def _session_ownership( - strategy_dir: Path, default_config: dict | None, num: int -) -> list[OwnedBot]: - """Bases a session owned and the instant it took each over, oldest first. - - Two sources, in order: - - 1. ``{session_dir}/owned_bots.json`` — the ledger [[FEAT-017]] writes, which - knows both the bases (a session may operate several) and the exact takeover - instant, whether the bot was deployed here or adopted after a restart. - 2. the legacy shim — a session predating the ledger resolves its single - ``bot_name`` as one owned bot ``since`` the session started, reproducing the - session-start tiling attribution used before the ledger existed. - - Empty for direct-executor strategies, whose per-session executor attribution - already stands and must not be touched. - """ - from condor.agents.ownership import read_owned - - owned = read_owned(find_session_dir(strategy_dir, num)) - if owned: - return owned - base = _session_bot_base(strategy_dir, default_config, num) - if not base: - return [] - start = _session_start_epoch(strategy_dir, num) - return [OwnedBot(base=base, origin="legacy", since=start, last_seen=start)] - - -def _owner_windows( - real_sessions: list, strategy_dir: Path, default_config: dict | None -) -> dict[str, list[tuple[float, Any, float]]]: - """``{base: [(since, session, until), …]}`` — owners, oldest takeover first. - - The windows a base's owners occupy tile ``[since_i, since_{i+1})`` and the last - one runs to now, so slicing over them reproduces the bot's whole cumulative with - no gap and no double count. Keyed per base rather than globally per session - number: two bases handed over at different moments never share a timeline. - - ``until`` is the instant the session released the bot, or ``0.0`` while it - still holds it. A released last window stops there rather than running to now, - which is the one case where the tiling deliberately leaves a gap: PnL a bot - earned with no session operating it belongs to no session. - """ - owners: dict[str, list[tuple[float, Any, float]]] = {} - for s in real_sessions: - for ob in _session_ownership(strategy_dir, default_config, s.session_num): - owners.setdefault(ob.base, []).append((ob.since, s, ob.until)) - for lst in owners.values(): - lst.sort(key=lambda t: (t[0], t[1].session_num)) - return owners - - -def _current_owner_bases( - strategy_dir: Path, - default_config: dict | None, - session_nums: list[int], - num: int, -) -> list[str]: - """Bases ``num`` is the CURRENT owner of — the last takeover by ``since``. - - A bot's live open positions belong to whoever operates it now, so this is the - gate for merging them into one session's view. Same rule - :func:`_apply_bot_mode_pnl` applies to live unrealized PnL, kept here as one - lookup over the same windows so the rollup and the per-session detail can - never disagree about who holds the open book. A session that released the bot - is not its current owner, so an ended session shows no live open book. - """ - last: dict[str, tuple[float, int, float]] = {} - for n in session_nums: - for ob in _session_ownership(strategy_dir, default_config, n): - if last.get(ob.base, (float("-inf"), -1, 0.0))[:2] <= (ob.since, n): - last[ob.base] = (ob.since, n, ob.until) - return sorted( - base for base, (_, owner, until) in last.items() if owner == num and until <= 0 - ) - - -async def _apply_bot_mode_pnl( - real_sessions: list, strategy_dir: Path, default_config: dict | None, client: Any -) -> None: - """Distribute each owned bot's PnL across the sessions that operated it. - - One rule covers deploy and handover: every owned bot is attributed by slicing - its history over ``[since, next_owner.since or now)``, where ``since`` is the - takeover instant the ownership ledger recorded. A bot the session *deployed* - has no history before its ``since``, so the general rule already hands it the - whole instance — the exact case falls out instead of needing its own branch. - - Live unrealized PnL, fees and open positions go to each base's LAST owner by - ``since`` — a lookup in the ledger where it used to be a ``max(session_num)`` - guess, so a new session that never adopted the bot no longer inherits its - open book. - - Works uniformly for single- and multi-controller bots (history sums controllers - per instance) and for a base re-launched under several instances. Strategies - whose sessions own no bot (direct-executor agents) are left untouched. - """ - from condor.fetchers.bot_performance import ( - bot_executor_rows, - fetch_all_bot_performance, - fetch_archived_instances, - fetch_base_histories, - resolve_bots, - slice_history, - ) - - if not client or not real_sessions: - return - owners = _owner_windows(real_sessions, strategy_dir, default_config) - bases = sorted(owners) - if not bases: - return # direct-executor strategy — nothing to attribute - - try: - all_perf = await fetch_all_bot_performance(client) - except Exception as e: - log.warning("bot perf fetch for %s failed: %s", strategy_dir.name, e) - return - - now = time.time() - # The oldest takeover across every base sets how far back the histories must - # reach; sampling resolution is chosen from it so no owner's window falls off - # the end of the retained rows. - earliest = min( - (since for lst in owners.values() for since, _, _ in lst if since > 0), - default=0.0, - ) - # Archived instances carry the realized PnL of every bot a session stopped — - # the normal end state of a finished session, and invisible in the live - # snapshot. Same universe the live agent's own view uses, so the dashboard and - # the tick loop cannot disagree about what a session earned. - archived = await fetch_archived_instances(client) - histories_by_base = await fetch_base_histories( - client, all_perf, bases, earliest, now, extra_names=archived - ) - - live = resolve_bots(all_perf, bases) - for base in bases: - window_owners = owners[base] - insts = histories_by_base.get(base, []) - - # Realized / volume / trades / fees: one window per owner, tiling the - # timeline. A released window (the session stopped and left the bot - # running) ends at its release instant, so PnL earned while no session - # was operating the bot is attributed to nobody instead of accruing to - # whoever happened to hold last. - sliced_fees = 0.0 - for i, (since, s, until) in enumerate(window_owners): - end = window_owners[i + 1][0] if i + 1 < len(window_owners) else now - if until > 0: - end = min(end, until) - if end <= since: - continue - realized, volume, trades, fees = slice_history(insts, since, end) - s.realized_pnl += realized - s.volume += volume - # Sliced closes are round-trip closes, so they are this session's - # trades AND its closed positions — the same two counters - # _merge_bot_perf bumps, so the session detail and this rollup report - # one number. - s.trade_count += int(round(trades)) - s.closed_count += int(round(trades)) - s.fees += fees - sliced_fees += fees - # Volume with no fee is a missing column, not a free trade. - if volume > 0 and fees == 0.0: - s.fees_known = False - s.total_pnl = s.realized_pnl + s.unrealized_pnl - - # Live unrealized + open positions → the base's current owner, unless it - # has released the bot: an ended session holds no open book. - bot = live.get(base) - if not bot: - continue - last_since, operator, last_until = window_owners[-1] - if last_until > 0: - continue - b_rows = bot_executor_rows(bot) - operator.unrealized_pnl += float(bot.get("unrealized_pnl_quote", 0) or 0) - # Fees come from the sliced history when the backend reports a cumulative - # figure. When it does not, that column is all zeros and the only fees - # available are the live open-position ones — attributed to the current - # operator as before rather than silently dropped. - if sliced_fees == 0.0: - operator.fees += float(bot.get("cum_fees_quote", 0) or 0) - operator.open_count += sum(1 for r in b_rows if r["status"] == "RUNNING") - operator.executors = list(operator.executors) + b_rows - operator.total_pnl = operator.realized_pnl + operator.unrealized_pnl +# ── Session-ownership PnL attribution ── +# The whole engine — session ownership resolution, owner-window tiling, the +# slice-and-merge fee rules, and the current-owner rule for the live open book — +# lives in condor.agents.attribution ([[ARCH-191]]), shared with the agent's own +# view (condor.agents.performance) so the dashboard and the tick loop compute +# from one implementation. This module keeps only HTTP concerns and the caches. async def _compute_strategy_performance( @@ -678,7 +463,7 @@ async def _compute_strategy_performance( # Per-session executor fetches stay bot-free (bot_names=None below) so closed # sessions can be frozen; bot-mode PnL is distributed per session afterward by - # _apply_bot_mode_pnl from the controller history, which handles both fixed and + # apply_bot_mode_pnl from the controller history, which handles both fixed and # runtime-named (per-session config) bots. sessions: list[AgentPerformanceModel] = [] if client and ids: @@ -769,7 +554,7 @@ async def _compute_strategy_performance( # bot is DISTRIBUTED — not duplicated — the totals below are a plain additive # sum of the rows and stay correct for both modes with no double counting. if client and real_sessions: - await _apply_bot_mode_pnl(real_sessions, strategy_dir, default_config, client) + await apply_bot_mode_pnl(real_sessions, strategy_dir, default_config, client) totals = { "total_pnl": sum(s.total_pnl for s in real_sessions), @@ -1724,7 +1509,7 @@ async def get_session_executors( # Bot-mode: the session operates named bots whose executors live in the bot # container, not the agent_id-keyed table. Merge the live positions of every # base this session CURRENTLY owns — the same last-owner-by-`since` rule - # _apply_bot_mode_pnl uses, so the two views never disagree. A session that + # apply_bot_mode_pnl uses, so the two views never disagree. A session that # handed its bot over shows only its own direct executors; the live open book # belongs to whoever operates the bot now. session_nums = [ @@ -1732,13 +1517,13 @@ async def get_session_executors( for _, n, k in enumerate_agent_ids(_runkey(slug, sslug), strategy.dir) if k == "session" ] - bot_names = _current_owner_bases( + bot_names = current_owner_bases( strategy.dir, strategy.default_config, session_nums, session_num ) # Slice the bot to this session's window for the same reason the rollup does: # merging the lifetime aggregate here made the session detail disagree with # the session's own row in the strategy list. - owned = _session_ownership(strategy.dir, strategy.default_config, session_num) + owned = session_ownership(strategy.dir, strategy.default_config, session_num) since = min((b.since for b in owned if b.since > 0), default=0.0) perf = await fetch_agent_performance( client, agent_id, bot_names=bot_names, since=since diff --git a/tests/test_bot_mode_session_resolution.py b/tests/test_bot_mode_session_resolution.py index 513495f7b..98d879c63 100644 --- a/tests/test_bot_mode_session_resolution.py +++ b/tests/test_bot_mode_session_resolution.py @@ -1,4 +1,4 @@ -"""Bot-mode session→bot_name resolution (`_session_bot_base`). +"""Bot-mode session→bot_name resolution (`session_bot_base`). The per-session PnL distribution and the operator's live-executor view share this one helper, so its fallback rules must hold for both: a non-empty per-session @@ -11,7 +11,7 @@ import yaml -from condor.web.routes.agents import _session_bot_base +from condor.agents.attribution import session_bot_base def _write_session(strategy_dir: Path, num: int, cfg: dict) -> None: @@ -24,7 +24,7 @@ def test_per_session_bot_name_wins(tmp_path): # A session that recorded its runtime-derived deploy name takes precedence. _write_session(tmp_path, 1, {"bot_name": "dn-CL-BRENTOIL-mm-20260724-182221"}) assert ( - _session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 1) + session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 1) == "dn-CL-BRENTOIL-mm-20260724-182221" ) @@ -32,23 +32,23 @@ def test_per_session_bot_name_wins(tmp_path): def test_empty_session_bot_name_falls_back_to_default(tmp_path): # Early session saved bot_name as '' — must still resolve to the shared bot. _write_session(tmp_path, 2, {"bot_name": ""}) - assert _session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 2) == "dn-mm" + assert session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 2) == "dn-mm" def test_missing_session_bot_name_falls_back_to_default(tmp_path): # Session config predates the key entirely. _write_session(tmp_path, 3, {"some_other": 1}) - assert _session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 3) == "dn-mm" + assert session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 3) == "dn-mm" def test_no_session_dir_uses_default(tmp_path): # A session_num with no on-disk config still maps to the strategy default. - assert _session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 99) == "dn-mm" + assert session_bot_base(tmp_path, {"bot_name": "dn-mm"}, 99) == "dn-mm" def test_direct_executor_strategy_resolves_empty(tmp_path): # No default and no per-session bot_name → direct-executor agent, nothing to # attribute; distribution and operator merge both correctly skip it. _write_session(tmp_path, 1, {"bot_name": ""}) - assert _session_bot_base(tmp_path, {}, 1) == "" - assert _session_bot_base(tmp_path, None, 99) == "" + assert session_bot_base(tmp_path, {}, 1) == "" + assert session_bot_base(tmp_path, None, 99) == "" diff --git a/tests/test_session_pnl_attribution.py b/tests/test_session_pnl_attribution.py index bbb80df34..b68ee2b90 100644 --- a/tests/test_session_pnl_attribution.py +++ b/tests/test_session_pnl_attribution.py @@ -12,12 +12,12 @@ import yaml -from condor.web.routes.agents import ( - AgentPerformanceModel, - _apply_bot_mode_pnl, - _current_owner_bases, - _session_ownership, +from condor.agents.attribution import ( + apply_bot_mode_pnl, + current_owner_bases, + session_ownership, ) +from condor.web.routes.agents import AgentPerformanceModel # ── Fixtures: on-disk sessions, with and without a ledger ── @@ -152,7 +152,7 @@ def test_handover_splits_pnl_at_the_takeover_instant(tmp_path): ) s1, s2 = _session(1), _session(2) - asyncio.run(_apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) assert s1.realized_pnl == 40.0 assert s2.realized_pnl == 60.0 @@ -184,7 +184,7 @@ def test_live_unrealized_goes_to_the_last_owner_not_the_newest_session(tmp_path) ) s1, s2 = _session(1), _session(2) - asyncio.run(_apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) assert s1.unrealized_pnl == 7.0 assert s1.open_count == 1 @@ -211,7 +211,7 @@ def test_session_owning_two_bots_sums_them_without_parent_folding(tmp_path): ) s1 = _session(1) - asyncio.run(_apply_bot_mode_pnl([s1], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, client)) assert s1.realized_pnl == 35.0 @@ -231,7 +231,7 @@ def test_parent_base_does_not_swallow_its_tagged_sibling(tmp_path): ) s1 = _session(1) - asyncio.run(_apply_bot_mode_pnl([s1], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, client)) # 35, not 60: the child instance is attributed to `-btc` alone. assert s1.realized_pnl == 35.0 @@ -252,7 +252,7 @@ def test_multi_controller_bot_rolls_up_to_one_session_figure(tmp_path): ) s1 = _session(1) - asyncio.run(_apply_bot_mode_pnl([s1], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, client)) assert s1.realized_pnl == 15.0 # 3 controllers × 5 @@ -270,7 +270,7 @@ def test_instance_history_is_fetched_once_per_instance(tmp_path): history={parent: [_hist_row(T3, 1.0)], child: [_hist_row(T3, 1.0)]}, ) - asyncio.run(_apply_bot_mode_pnl([_session(1), _session(2)], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([_session(1), _session(2)], tmp_path, None, client)) # Two instances, four (base, owner) windows → still exactly two fetches. assert sorted(client.history_calls) == sorted([parent, child]) @@ -405,8 +405,8 @@ def test_ledgerless_sessions_keep_session_start_tiling(tmp_path): _write_session(tmp_path, 1, {"bot_name": "dn-mm"}) _write_session(tmp_path, 2, {"bot_name": "dn-mm"}) - owned1 = _session_ownership(tmp_path, {"bot_name": "dn-mm"}, 1) - owned2 = _session_ownership(tmp_path, {"bot_name": "dn-mm"}, 2) + owned1 = session_ownership(tmp_path, {"bot_name": "dn-mm"}, 1) + owned2 = session_ownership(tmp_path, {"bot_name": "dn-mm"}, 2) assert [o.base for o in owned1] == ["dn-mm"] assert [o.origin for o in owned1] == ["legacy"] # Windows come from session start times, and session 2 started after 1. @@ -418,7 +418,7 @@ def test_ledgerless_sessions_keep_session_start_tiling(tmp_path): history={inst: [_hist_row(T0, 0.0), _hist_row(T3, 50.0)]}, ) s1, s2 = _session(1), _session(2) - asyncio.run(_apply_bot_mode_pnl([s1, s2], tmp_path, {"bot_name": "dn-mm"}, client)) + asyncio.run(apply_bot_mode_pnl([s1, s2], tmp_path, {"bot_name": "dn-mm"}, client)) # Both sessions started after T3 (they were written now), so the whole history # predates session 1's window and neither is credited — exactly what the # session-start tiling did before the ledger existed. What matters is the @@ -429,27 +429,27 @@ def test_ledgerless_sessions_keep_session_start_tiling(tmp_path): def test_ledger_wins_over_the_legacy_config_name(tmp_path): sd = _write_session(tmp_path, 1, {"bot_name": "dn-mm"}) _write_ledger(sd, {"ns-other": 123.0}) - owned = _session_ownership(tmp_path, {"bot_name": "dn-mm"}, 1) + owned = session_ownership(tmp_path, {"bot_name": "dn-mm"}, 1) assert [(o.base, o.since) for o in owned] == [("ns-other", 123.0)] def test_direct_executor_strategy_owns_nothing(tmp_path): _write_session(tmp_path, 1, {"bot_name": ""}) - assert _session_ownership(tmp_path, {}, 1) == [] + assert session_ownership(tmp_path, {}, 1) == [] # ── Current-owner lookup (per-session executors view) ── -def test_current_owner_bases_follows_the_ledger_not_the_session_number(tmp_path): +def testcurrent_owner_bases_follows_the_ledger_not_the_session_number(tmp_path): sd1 = _write_session(tmp_path, 1) sd2 = _write_session(tmp_path, 2) _write_ledger(sd1, {"ns-a": 100.0, "ns-b": 100.0}) _write_ledger(sd2, {"ns-a": 200.0}) # session 2 adopted only ns-a nums = [1, 2] - assert _current_owner_bases(tmp_path, None, nums, 1) == ["ns-b"] - assert _current_owner_bases(tmp_path, None, nums, 2) == ["ns-a"] + assert current_owner_bases(tmp_path, None, nums, 1) == ["ns-b"] + assert current_owner_bases(tmp_path, None, nums, 2) == ["ns-a"] # ── Whole-server snapshot is fetched once per rollup, not once per strategy ── @@ -458,7 +458,7 @@ def test_current_owner_bases_follows_the_ledger_not_the_session_number(tmp_path) def test_rollup_fans_out_one_snapshot_call_for_all_strategies(tmp_path): """N strategies on one server share a single whole-server snapshot fetch. - ``list_agents`` gathers every strategy's ``_apply_bot_mode_pnl`` at once, and + ``list_agents`` gathers every strategy's ``apply_bot_mode_pnl`` at once, and ``get_latest_controller_performance()`` returns the same whole-server payload to each of them — so the fan-out must collapse to one round-trip. """ @@ -492,7 +492,7 @@ async def get_latest_controller_performance(self): async def _go(): await asyncio.gather( *[ - _apply_bot_mode_pnl(sessions, sdir, None, client) + apply_bot_mode_pnl(sessions, sdir, None, client) for sdir, sessions in strategies ] ) @@ -538,7 +538,7 @@ def test_long_lived_bot_does_not_dump_old_pnl_on_one_session(tmp_path): ) s1, s2, s3 = _session(1), _session(2), _session(3) - asyncio.run(_apply_bot_mode_pnl([s1, s2, s3], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1, s2, s3], tmp_path, None, client)) # Each session keeps only what it earned; none is starved and none inherits. assert s1.realized_pnl == 40.0 # days 0–4 @@ -574,7 +574,7 @@ def test_fees_are_sliced_per_session_not_dumped_on_the_last_owner(tmp_path): ) s1, s2 = _session(1), _session(2) - asyncio.run(_apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1, s2], tmp_path, None, client)) assert s1.fees == 4.0 assert s2.fees == 5.0 @@ -600,7 +600,7 @@ def test_fees_fall_back_to_live_figure_when_history_has_no_fee_column(tmp_path): ) s1 = _session(1) - asyncio.run(_apply_bot_mode_pnl([s1], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, client)) assert s1.fees == 3.5 @@ -628,7 +628,7 @@ def test_released_session_stops_accruing_a_surviving_bot(tmp_path): ) s1 = _session(1) - asyncio.run(_apply_bot_mode_pnl([s1], tmp_path, None, client)) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, client)) assert s1.realized_pnl == 40.0 # only up to the release assert s1.unrealized_pnl == 0.0 # an ended session holds no open book @@ -636,11 +636,11 @@ def test_released_session_stops_accruing_a_surviving_bot(tmp_path): def test_released_session_is_not_the_current_owner(tmp_path): - """_current_owner_bases and the slicing agree that a released bot is unowned.""" + """current_owner_bases and the slicing agree that a released bot is unowned.""" sd1 = _write_session(tmp_path, 1) _write_ledger(sd1, {"ns-bot": _epoch(T0)}, until={"ns-bot": _epoch(T2)}) - assert _current_owner_bases(tmp_path, None, [1], 1) == [] + assert current_owner_bases(tmp_path, None, [1], 1) == [] def test_release_is_idempotent_and_reopens_on_further_use(tmp_path): @@ -688,3 +688,65 @@ def test_release_preserves_namespace_for_a_ledger_opened_without_one(tmp_path): assert reopened.namespace == "ns" # not blanked by the release write assert reopened.declared == ["legacy-bot"] assert read_owned(sd)[0].until == 200.0 + + +# ── One engine, two surfaces (ARCH-191) ── + + +def test_rollup_and_agent_view_agree_on_an_adopted_bot(): + """The dashboard rollup and the agent's own fetch share one attribution engine. + + A session that adopts a long-running bot must be credited the same + realized / volume / trades / fees on both surfaces: the web strategy rollup + (``apply_bot_mode_pnl``) and the agent's own ``fetch_agent_performance``. + Both now fold through ``condor.agents.attribution``, so agreement is + structural — this test pins the contract over a synthetic history. + """ + import tempfile + from types import SimpleNamespace + + from condor.agents.performance import fetch_agent_performance + + with tempfile.TemporaryDirectory() as td: + tmp_path = Path(td) + sd1 = _write_session(tmp_path, 1) + # The bot ran since T0; this session only took it over at T2. + _write_ledger(sd1, {"ns-bot": _epoch(T2)}) + + inst = "ns-bot-20260701-000000" + closes = {"CloseType.TAKE_PROFIT": 3} + history = { + inst: [ + _hist_row(T0, 0.0, cum_volume=0.0, cum_fees=0.0), + _hist_row(T2, 40.0, cum_volume=4000.0, cum_fees=4.0), + _hist_row(T3, 100.0, cum_volume=9000.0, cum_fees=9.0, closes=closes), + ] + } + snapshots = [_snap(inst, T3, realized=100.0, unrealized=7.0)] + + # Web rollup surface. + rollup_client = _FakeClient(snapshots=snapshots, history=history) + s1 = _session(1) + asyncio.run(apply_bot_mode_pnl([s1], tmp_path, None, rollup_client)) + + # Agent's own surface, over the same ownership window. + async def _no_executors(**_kw): + return [] + + agent_client = _FakeClient(snapshots=snapshots, history=history) + agent_client.executors = SimpleNamespace(search_executors=_no_executors) + owned = session_ownership(tmp_path, None, 1) + since = min(b.since for b in owned) + detail = asyncio.run( + fetch_agent_performance( + agent_client, "a_1", bot_names=[b.base for b in owned], since=since + ) + ) + + # Post-takeover only — the inherited 40/4000/4.0 belongs to nobody here. + assert s1.realized_pnl == detail.realized_pnl == 60.0 + assert s1.volume == detail.volume == 5000.0 + assert s1.fees == detail.fees == 5.0 + assert s1.trade_count == detail.trade_count == 3 + assert s1.unrealized_pnl == detail.unrealized_pnl == 7.0 + assert s1.fees_known and detail.fees_known From 196c0fb2b344bb62fa1237944d4050000e87fa65 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 21:12:01 +0300 Subject: [PATCH 44/57] (refactor) split WebSocketManager god class: extract candle and Hummingbot stream families into condor/web/streams --- condor/web/streams/__init__.py | 47 + condor/web/streams/candles.py | 720 ++++++++++++++++ condor/web/streams/hummingbot_ws.py | 599 +++++++++++++ condor/web/ws_manager.py | 1237 +-------------------------- 4 files changed, 1394 insertions(+), 1209 deletions(-) create mode 100644 condor/web/streams/__init__.py create mode 100644 condor/web/streams/candles.py create mode 100644 condor/web/streams/hummingbot_ws.py diff --git a/condor/web/streams/__init__.py b/condor/web/streams/__init__.py new file mode 100644 index 000000000..1c3b53512 --- /dev/null +++ b/condor/web/streams/__init__.py @@ -0,0 +1,47 @@ +"""Upstream stream implementations behind the ``WebSocketManager``. + +Each module in this package owns one stream family: + +- ``candles``: the candle subsystem — per-channel buffer, backend WS stream, + REST poll fallback, historical backfill and idle-buffer cleanup. +- ``hummingbot_ws``: the streams bridged from the Hummingbot backend WS + endpoints (trades, order book, executors, bots, positions, performance) + plus the controller-performance REST poll. + +The stream code is packaged as mixins over a small host surface +(:class:`StreamHost`) that ``WebSocketManager`` implements: broadcasting, +subscriber checks, per-connection sends and the shared one-shot task tracker. +A unit test can drive a mixin with a stub host instead of the full manager, +and ``WebSocketManager`` itself keeps only connection bookkeeping, the SDS +bridge and the stream lifecycle registry. +""" + +from __future__ import annotations + +from typing import Any, Protocol + +from condor.asyncutil import TaskSet + + +class StreamHost(Protocol): + """What the stream mixins require of the class they are mixed into. + + ``WebSocketManager`` is the one production implementation; the protocol + exists so each stream family can be unit-tested against a stub. + """ + + _last_data: dict[str, Any] + _oneshot_tasks: TaskSet + + async def broadcast(self, channel: str, data: Any) -> None: ... + + async def _broadcast_update(self, channel: str, data: Any) -> None: ... + + async def _send(self, conn: Any, channel: str, data: Any) -> None: ... + + def _has_subscribers(self, channel: str) -> bool: ... + + def _ensure_stream(self, prefix: str, channel: str) -> None: ... + + +__all__ = ["StreamHost"] diff --git a/condor/web/streams/candles.py b/condor/web/streams/candles.py new file mode 100644 index 000000000..ac6534d86 --- /dev/null +++ b/condor/web/streams/candles.py @@ -0,0 +1,720 @@ +"""Candle streaming subsystem: buffer, backend WS stream, REST poll fallback, +historical backfill and idle-buffer cleanup. + +``CandleStreamsMixin`` is mixed into ``WebSocketManager`` (the +:class:`~condor.web.streams.StreamHost` implementation): it calls the host's +``broadcast`` / ``_send`` / ``_has_subscribers`` / ``_ensure_stream`` and +tracks its fire-and-forget work on the host's ``_oneshot_tasks``. The candle +state it drives (``_candle_buffers``, ``_candle_tasks``, +``_candle_poll_tasks``, teardown timers and staleness bookkeeping) is +initialized by the host's ``__init__``. + +Only this module touches ``_CandleBuffer`` internals; everything else goes +through its accessors. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import math +import time +from typing import TYPE_CHECKING, Any + +from condor import dex_candles +from condor.asyncutil import TaskSet +from condor.fetchers.market_data import fetch_historical_candles, normalize_candle + +if TYPE_CHECKING: + from condor.web.ws_manager import _Connection + +logger = logging.getLogger(__name__) + +# Interval string -> seconds for buffer sizing +_INTERVAL_SECONDS: dict[str, int] = { + "1s": 1, + "5s": 5, + "15s": 15, + "30s": 30, + "1m": 60, + "3m": 180, + "5m": 300, + "15m": 900, + "30m": 1800, + "1h": 3600, + "2h": 7200, + "4h": 14400, + "1d": 86400, + "1w": 604800, +} + +# Auto-cleanup candle buffers unused for this long +_CANDLE_BUFFER_IDLE_TTL = 600 # 10 minutes + + +def _coerce_duration(value: object) -> int | None: + """Coerce a client-supplied candle duration to a positive int (seconds). + + WS messages are untrusted input: a non-numeric value must not raise, or the + ValueError would escape handle_message and tear down the client's entire + multiplexed connection. Returns None for missing/invalid/non-positive values. + """ + if value is None or isinstance(value, bool): + return None + try: + duration = int(float(value)) + except (TypeError, ValueError): + logger.warning("Ignoring invalid candle duration from client: %r", value) + return None + return duration if duration > 0 else None + + +def parse_candle_channel(channel: str) -> tuple[str, str, str, str, str | None] | None: + """Split a candle channel into its parts, or ``None`` if it is malformed. + + The channel is ``candles:{server}:{connector}:{pair}:{interval}[:{pool_address}]``. + The pool address is an **optional sixth segment**, set by the DEX page so a + chart is pinned to the exact pool the user opened instead of whichever pool + ``dex_candles._resolve_pool`` judges deepest for the pair. A five-part channel + (every CLOB chart) parses to ``pool_address=None`` and is byte-identical to + what it was before the segment existed. + + Every consumer must go through here: unpacking five names off ``split(":")`` + raises ``ValueError`` on a six-part channel, and each of the three call sites + fails silently in a different way (a stream that never backfills, a backfill + that never ticks). + """ + parts = channel.split(":") + if len(parts) < 5: + return None + server_name, connector, pair, interval = parts[1:5] + pool_address = parts[5] if len(parts) > 5 else None + return server_name, connector, pair, interval, pool_address or None + + +class _CandleBuffer: + """Per-channel candle buffer with dynamic sizing based on interval + duration.""" + + __slots__ = ("interval", "_data", "_max_size", "last_accessed") + + def __init__(self, interval: str, duration_seconds: int = 3600): + self.interval = interval + self._data: dict[float, dict] = {} + self._max_size: int = 200 + self.last_accessed: float = time.monotonic() + self.set_duration(duration_seconds) + + def set_duration(self, duration_seconds: int) -> int: + """Resize buffer for the given duration. Returns the new max size.""" + interval_sec = _INTERVAL_SECONDS.get(self.interval, 60) + needed = math.ceil(duration_seconds / interval_sec) + new_max = max(needed, 200) # minimum 200 + old_max = self._max_size + self._max_size = new_max + self.last_accessed = time.monotonic() + self._evict() + if new_max != old_max: + logger.debug( + "Candle buffer resized %s: %d -> %d (duration=%ds)", + self.interval, + old_max, + new_max, + duration_seconds, + ) + return new_max + + def upsert(self, candle: dict) -> None: + self._data[candle["timestamp"]] = candle + self._evict() + + def upsert_many(self, candles: list[dict]) -> None: + for c in candles: + self._data[c["timestamp"]] = c + self._evict() + + def get_sorted(self) -> list[dict]: + self.last_accessed = time.monotonic() + return sorted(self._data.values(), key=lambda c: c["timestamp"]) + + def get(self, timestamp: float) -> dict | None: + """The buffered candle at ``timestamp``, or ``None``.""" + return self._data.get(timestamp) + + def _evict(self) -> None: + excess = len(self._data) - self._max_size + if excess <= 0: + return + # Drop the `excess` oldest timestamps in a single O(n log n) pass + # instead of calling min() (O(n)) per evicted candle. + for ts in sorted(self._data)[:excess]: + del self._data[ts] + + @property + def size(self) -> int: + return len(self._data) + + @property + def max_size(self) -> int: + return self._max_size + + @property + def latest_timestamp(self) -> float: + """Timestamp of the newest buffered candle, or 0 when empty.""" + return max(self._data) if self._data else 0 + + @property + def needs_backfill(self) -> bool: + """True if buffer has room for significantly more candles.""" + return len(self._data) < self._max_size * 0.5 + + +class CandleStreamsMixin: + """Candle stream family: subscribe handling, backfill, backend WS stream, + REST poll fallback, deferred teardown and idle-buffer cleanup.""" + + _CANDLE_KEEP_ALIVE = 300 # 5-minute grace period before tearing down candle streams + + if TYPE_CHECKING: + # Candle state, initialized by WebSocketManager.__init__. + _candle_tasks: dict[str, asyncio.Task] + _candle_poll_tasks: dict[str, asyncio.Task] + _candle_buffers: dict[str, _CandleBuffer] + _candle_teardown_timers: dict[str, asyncio.TimerHandle] + _last_candle_ws_update: dict[str, float] + _candle_first_msg_logged: set[str] + _connections: list[Any] + # Host surface (see condor.web.streams.StreamHost). + _last_data: dict[str, Any] + _oneshot_tasks: TaskSet + + async def broadcast(self, channel: str, data: Any) -> None: ... + + async def _send(self, conn: Any, channel: str, data: Any) -> None: ... + + def _has_subscribers(self, channel: str) -> bool: ... + + def _ensure_stream(self, prefix: str, channel: str) -> None: ... + + # Provided by HummingbotStreamsMixin on the composed manager. + @staticmethod + def _is_permanent_ws_error(error_str: str) -> bool: ... + + @staticmethod + def _normalize_candle(c: Any) -> dict | None: + """Normalize a candle from any format to a uniform dict with float values.""" + return normalize_candle(c) + + # -- Subscribe handling -- + + async def _handle_candle_subscribe( + self, conn: "_Connection", channel: str, duration: object + ) -> None: + """Handle candle channel subscription with optional duration.""" + parts = channel.split(":") + if len(parts) < 5: + return + interval = parts[4] + dur = _coerce_duration(duration) or 3 * 86400 # default 3 days + + # Cancel any pending teardown — a subscriber just came back + timer = self._candle_teardown_timers.pop(channel, None) + if timer is not None: + timer.cancel() + logger.debug("Cancelled pending candle teardown for %s", channel) + + buf = self._candle_buffers.get(channel) + if buf is None: + buf = _CandleBuffer(interval, dur) + self._candle_buffers[channel] = buf + else: + # Expand buffer if this client needs more + if dur > 0: + old_max = buf.max_size + buf.set_duration(dur) + if buf.max_size > old_max and buf.needs_backfill: + self._oneshot_tasks.track( + asyncio.create_task( + self._backfill_candles(channel), + name=f"backfill:{channel}", + ) + ) + + # Send buffered candles as initial snapshot + sorted_candles = buf.get_sorted() + if sorted_candles: + await self._send(conn, channel, {"type": "candles", "data": sorted_candles}) + + # If the stream task is still running (kept alive during grace period), skip restart + self._ensure_stream("candles", channel) + + async def _handle_candle_duration_change( + self, conn: "_Connection", channel: str, duration: int + ) -> None: + """Handle duration change for an existing candle subscription. + + Only grows the buffer — never shrinks. The frontend manages its own + display window; the backend just ensures enough history is buffered. + """ + buf = self._candle_buffers.get(channel) + if buf is None: + return + old_max = buf.max_size + # Only expand — skip if requested duration would shrink the buffer + interval_sec = _INTERVAL_SECONDS.get(buf.interval, 60) + needed = max(math.ceil(duration / interval_sec), 200) + if needed <= old_max: + return + buf.set_duration(duration) + if buf.needs_backfill: + await self._backfill_candles(channel) + # Broadcast updated snapshot to ALL subscribers on this channel + sorted_candles = buf.get_sorted() + if sorted_candles: + await self.broadcast(channel, {"type": "candles", "data": sorted_candles}) + + async def _backfill_candles(self, channel: str) -> None: + """Fetch historical candles to fill the buffer gap.""" + parsed = parse_candle_channel(channel) + if parsed is None: + return + server_name, connector, pair, interval, pool_address = parsed + buf = self._candle_buffers.get(channel) + if buf is None: + return + + from config_manager import get_config_manager + + cm = get_config_manager() + try: + interval_sec = _INTERVAL_SECONDS.get(interval, 60) + end_time = int(time.time()) + start_time = end_time - (buf.max_size * interval_sec) + + logger.info( + "Backfilling candles for %s: need %d, have %d, fetching %ds-%ds", + channel, + buf.max_size, + buf.size, + start_time, + end_time, + ) + + if dex_candles.uses_gecko_candles(connector): + candles = await dex_candles.fetch_dex_candles( + connector, pool_address, pair, interval, start_time, end_time + ) + if candles: + buf.upsert_many(candles) + logger.info( + "Backfilled %d GeckoTerminal candles for %s (buffer: %d/%d)", + len(candles), + channel, + buf.size, + buf.max_size, + ) + return + + client = await cm.get_client(server_name) + candles = await fetch_historical_candles( + client, + connector, + pair, + interval, + start_time=start_time, + end_time=end_time, + limit=min(buf.max_size, 5000), + ) + if candles: + buf.upsert_many(candles) + logger.info( + "Backfilled %d candles for %s (buffer: %d/%d)", + len(candles), + channel, + buf.size, + buf.max_size, + ) + except Exception as e: + logger.warning("Candle backfill failed for %s: %s", channel, e) + + # -- Deferred teardown -- + + def _maybe_stop_candle_stream(self, channel: str) -> None: + # If subscribers still exist, cancel any pending teardown and return + if self._has_subscribers(channel): + timer = self._candle_teardown_timers.pop(channel, None) + if timer is not None: + timer.cancel() + return + + # Already have a pending teardown scheduled — nothing to do + if channel in self._candle_teardown_timers: + return + + # Schedule deferred teardown + loop = asyncio.get_event_loop() + handle = loop.call_later( + self._CANDLE_KEEP_ALIVE, + self._deferred_stop_candle_stream, + channel, + ) + self._candle_teardown_timers[channel] = handle + logger.info( + "Scheduled candle stream teardown for %s in %ds", + channel, + self._CANDLE_KEEP_ALIVE, + ) + + def _deferred_stop_candle_stream(self, channel: str) -> None: + """Actually tear down a candle stream after the grace period.""" + self._candle_teardown_timers.pop(channel, None) + + # Re-check: subscribers may have appeared during the grace period + if self._has_subscribers(channel): + logger.debug( + "Candle teardown cancelled — subscribers returned for %s", channel + ) + return + + task = self._candle_tasks.pop(channel, None) + if task and not task.done(): + task.cancel() + logger.info("Deferred candle stream teardown completed for %s", channel) + poll_task = self._candle_poll_tasks.pop(channel, None) + if poll_task and not poll_task.done(): + poll_task.cancel() + self._last_candle_ws_update.pop(channel, None) + self._candle_first_msg_logged.discard(channel) + # Snapshot-on-resubscribe is served by _candle_buffers, not _last_data + self._last_data.pop(channel, None) + # NOTE: candle buffer is NOT deleted — the existing idle cleanup loop handles that + + # -- Backend WS stream -- + + async def _candle_stream(self, channel: str) -> None: + parsed = parse_candle_channel(channel) + if parsed is None: + return + server_name, connector, pair, interval, pool_address = parsed + + from config_manager import get_config_manager + + cm = get_config_manager() + backoff = 5 + + # Start REST poll fallback alongside the WS stream + self._ensure_candle_poll_fallback(channel) + + if dex_candles.uses_gecko_candles(connector): + # No CandlesFactory feed to subscribe to (xrpl, DEX networks): the + # poll loop above is the whole stream. Subscribing anyway would just + # reconnect-loop on an error the API can never stop returning. + logger.info( + "Candle stream for %s is GeckoTerminal-polled (no API feed)", channel + ) + return + + while True: + try: + client = await cm.get_client(server_name) + async with client.ws.market_data() as ws: + await ws.subscribe_candles( + connector, + pair, + interval=interval, + max_records=100, + update_interval=1.0, + ) + logger.info("Candle WS subscribed: %s", channel) + backoff = 5 # Reset on successful connection + async for msg in ws: + # Log first message per channel at INFO for diagnostics + if channel not in self._candle_first_msg_logged: + self._candle_first_msg_logged.add(channel) + logger.info( + "Candle WS first message for %s: %s", + channel, + ( + json.dumps(msg)[:500] + if isinstance(msg, dict) + else type(msg).__name__ + ), + ) + else: + logger.debug( + "Candle WS raw msg keys for %s: %s", + channel, + ( + list(msg.keys()) + if isinstance(msg, dict) + else type(msg).__name__ + ), + ) + + msg_type = msg.get("type") + if msg_type == "candle_update": + raw = msg.get("data") + candle = self._normalize_candle(raw) if raw else None + if candle: + self._last_candle_ws_update[channel] = time.monotonic() + self._upsert_candle_buffer(channel, candle) + await self.broadcast( + channel, + {"type": "candle_update", "candle": candle}, + ) + elif msg_type == "candles": + raw_list = msg.get("data") or [] + candles = [ + c + for r in raw_list + if (c := self._normalize_candle(r)) is not None + ] + if candles: + self._last_candle_ws_update[channel] = time.monotonic() + self._upsert_candle_buffer_many(channel, candles) + await self.broadcast( + channel, + {"type": "candles", "data": candles}, + ) + elif msg_type in ("heartbeat", "subscribed"): + continue + elif msg_type == "error": + error_msg = msg.get("message", "unknown error") + logger.warning( + "Candle stream error for %s: %s — continuing", + channel, + error_msg, + ) + await self.broadcast( + channel, + { + "type": "error", + "message": f"Stream error: {error_msg}", + }, + ) + # Don't break — the WS may still be alive. + # If truly dead, next recv raises and we reconnect. + continue + else: + logger.info( + "Candle stream unrecognized msg type for %s: type=%s keys=%s", + channel, + msg_type, + ( + list(msg.keys()) + if isinstance(msg, dict) + else type(msg).__name__ + ), + ) + + except asyncio.CancelledError: + return + except Exception as e: + error_str = str(e) + # Detect permanent failures — don't retry (issue #134, see + # _is_permanent_ws_error). + if self._is_permanent_ws_error(error_str): + logger.warning( + "Candle stream permanent error for %s: %s — giving up", + channel, + e, + ) + await self.broadcast( + channel, + {"type": "error", "message": f"Stream failed: {error_str}"}, + ) + # Stop the paired REST poll fallback — nothing valid to poll. + poll_task = self._candle_poll_tasks.pop(channel, None) + if poll_task and not poll_task.done(): + poll_task.cancel() + return + + logger.warning( + "Candle stream error for %s: %s, reconnecting in %ds...", + channel, + e, + backoff, + ) + await self.broadcast( + channel, + { + "type": "error", + "message": f"Connection lost, retrying in {backoff}s", + }, + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 60) + + # -- REST poll fallback -- + + def _ensure_candle_poll_fallback(self, channel: str) -> None: + """Start a REST poll fallback task for a candle channel if not already running.""" + if ( + channel in self._candle_poll_tasks + and not self._candle_poll_tasks[channel].done() + ): + return + self._candle_poll_tasks[channel] = asyncio.create_task( + self._candle_poll_fallback(channel) + ) + + async def _candle_poll_fallback(self, channel: str) -> None: + """Periodically poll REST for candles when WS stream goes silent. + + For a GeckoTerminal-backed connector there is no WS stream to fall back + from — this loop *is* the feed, so it polls from the first tick and never + waits for staleness. + """ + parsed = parse_candle_channel(channel) + if parsed is None: + return + server_name, connector, pair, interval, pool_address = parsed + interval_sec = _INTERVAL_SECONDS.get(interval, 60) + gecko = dex_candles.uses_gecko_candles(connector) + # How long without a WS update before we consider it stale + stale_threshold = max(interval_sec, 15) + # How often to poll REST once stale (keep the last candle fresh). Gecko + # polls stay well clear of its per-minute rate limit — every chart on the + # dashboard shares one budget — while still refreshing a forming candle. + poll_interval = ( + max(min(interval_sec // 2, 60), 30) if gecko else min(interval_sec, 10) + ) + was_stale = gecko + + from config_manager import get_config_manager + + try: + while True: + await asyncio.sleep(poll_interval if was_stale else stale_threshold) + + last_update = self._last_candle_ws_update.get(channel) + if ( + last_update is not None + and (time.monotonic() - last_update) <= stale_threshold + ): + if was_stale: + logger.info( + "Candle WS stream resumed for %s, stopping REST fallback polling", + channel, + ) + was_stale = False + continue + + # Stream is stale — poll REST + if not was_stale: + logger.warning( + "Candle WS stream appears stale for %s (threshold=%ds), polling REST fallback", + channel, + stale_threshold, + ) + was_stale = True + + try: + now = int(time.time()) + if gecko: + candles = await dex_candles.fetch_dex_candles( + connector, + pool_address, + pair, + interval, + now - interval_sec * 5, + now, + # The shared OHLCV cache holds a pool for minutes, + # long enough to freeze a live chart. + use_cache=False, + ) + else: + cm = get_config_manager() + client = await cm.get_client(server_name) + # No `limit`: this poll only wants the fresh tail of the + # live range, so an empty historical answer stays empty + # rather than pulling a full unrelated window. + candles = await fetch_historical_candles( + client, + connector, + pair, + interval, + start_time=now - interval_sec * 5, + end_time=now, + ) + if candles: + buf = self._candle_buffers.get(channel) + # Broadcast if we have newer candles OR if the latest + # candle's OHLCV changed (same timestamp, updated values) + newest_buf_ts = buf.latest_timestamp if buf else 0 + newest_poll_ts = max(c["timestamp"] for c in candles) + changed = newest_poll_ts > newest_buf_ts + if not changed and buf: + old = buf.get(newest_poll_ts) + if old is not None: + # Same timestamp — check if OHLCV actually changed + new = next( + c + for c in candles + if c["timestamp"] == newest_poll_ts + ) + changed = any( + old.get(k) != new.get(k) + for k in ("open", "high", "low", "close", "volume") + ) + if changed: + self._upsert_candle_buffer_many(channel, candles) + await self.broadcast( + channel, + {"type": "candles", "data": candles}, + ) + logger.debug( + "REST fallback delivered %d candles for %s", + len(candles), + channel, + ) + except Exception as e: + logger.debug("REST candle poll failed for %s: %s", channel, e) + + except asyncio.CancelledError: + return + + # -- Buffer helpers -- + + def _get_or_create_candle_buffer(self, channel: str) -> _CandleBuffer: + """Get existing buffer or create with default duration.""" + buf = self._candle_buffers.get(channel) + if buf is None: + parts = channel.split(":") + interval = parts[4] if len(parts) >= 5 else "1m" + buf = _CandleBuffer(interval, 3 * 86400) # default 3 days + self._candle_buffers[channel] = buf + return buf + + def _upsert_candle_buffer(self, channel: str, candle: dict) -> None: + """Upsert a single candle into the per-channel buffer.""" + buf = self._get_or_create_candle_buffer(channel) + buf.upsert(candle) + + def _upsert_candle_buffer_many(self, channel: str, candles: list[dict]) -> None: + """Upsert multiple candles into the per-channel buffer.""" + buf = self._get_or_create_candle_buffer(channel) + buf.upsert_many(candles) + + async def _candle_buffer_cleanup_loop(self) -> None: + """Periodically remove candle buffers that haven't been accessed.""" + try: + while True: + await asyncio.sleep(60) # check every minute + now = time.monotonic() + stale = [ + ch + for ch, buf in self._candle_buffers.items() + if (now - buf.last_accessed) > _CANDLE_BUFFER_IDLE_TTL + and not any(ch in c.channels for c in self._connections) + ] + for ch in stale: + buf = self._candle_buffers.pop(ch, None) + if buf: + logger.info( + "Cleaned up idle candle buffer: %s (%d candles)", + ch, + buf.size, + ) + except asyncio.CancelledError: + return diff --git a/condor/web/streams/hummingbot_ws.py b/condor/web/streams/hummingbot_ws.py new file mode 100644 index 000000000..1056505e0 --- /dev/null +++ b/condor/web/streams/hummingbot_ws.py @@ -0,0 +1,599 @@ +"""Streams bridged from the Hummingbot backend WS endpoints, plus the +controller-performance REST poll. + +``HummingbotStreamsMixin`` is mixed into ``WebSocketManager`` (the +:class:`~condor.web.streams.StreamHost` implementation). It owns the shared +connect/reconnect skeleton (``_run_ws_stream``) and every per-protocol +handler built on it: trades and order book (``/ws/market-data``), executors, +bots, positions and performance (``/ws/executors``), and the 30s +controller-performance poll. Channel names and message shapes are part of the +dashboard's wire contract — the frontend depends on them byte-for-byte. +""" + +from __future__ import annotations + +import asyncio +import logging +from typing import TYPE_CHECKING, Any, Awaitable, Callable + +logger = logging.getLogger(__name__) + + +class HummingbotStreamsMixin: + """Upstream Hummingbot WS stream family and its shared reconnect skeleton.""" + + if TYPE_CHECKING: + # Stream task state, initialized by WebSocketManager.__init__. + _controller_perf_tasks: dict[str, asyncio.Task] + # Host surface (see condor.web.streams.StreamHost). + _last_data: dict[str, Any] + + async def broadcast(self, channel: str, data: Any) -> None: ... + + async def _broadcast_update(self, channel: str, data: Any) -> None: ... + + def _has_subscribers(self, channel: str) -> bool: ... + + # -- Transforms (REST-shape parity for WS broadcasts) -- + + @staticmethod + def _transform_executors(raw_data: Any) -> list[dict]: + """Transform raw executor data to ExecutorInfo-compatible dicts for WS broadcast.""" + from condor.fetchers.executors import extract_executors_list + from condor.web.models import ExecutorInfo + + executors_list = extract_executors_list(raw_data) + result = [] + for ex in executors_list: + info = ExecutorInfo.from_raw(ex) + if info: + result.append(info.model_dump()) + return result + + @staticmethod + def _transform_bots(raw_data: Any) -> dict: + """Transform raw BOTS_STATUS data to BotsPageResponse-compatible dict for WS broadcast.""" + from condor.fetchers.bots import build_bots_page + + return build_bots_page(raw_data) + + @staticmethod + def _overlay_stopping_state(server_name: str, data: dict) -> None: + """Apply transitional 'stopping' state to WS broadcast data.""" + from condor.web.routes.bots import overlay_stopping_state + + overlay_stopping_state( + server_name, data.get("controllers", []), data.get("bots", []) + ) + + # -- Shared skeleton -- + + @staticmethod + def _is_permanent_ws_error(error_str: str) -> bool: + """True if a stream error will never succeed on retry. + + Besides auth/not-found (401/403/404), an invalid trading pair (wrong + symbol for this connector) will never become valid, so retrying + forever just spams logs and hammers the exchange until Condor + restarts (issue #134). + """ + lowered = error_str.lower() + return ( + any(code in error_str for code in ("401", "403", "404")) + or "appears to be invalid" in lowered + or "invalid symbol" in lowered + ) + + async def _run_ws_stream( + self, + channel: str, + server_name: str, + *, + label: str, + open_ws: Callable[[Any], Any], + subscribe: Callable[[Any], Awaitable[None]], + on_message: Callable[[dict], Awaitable[None]], + backoff_on_empty_close: bool = False, + ) -> None: + """Shared connect/reconnect skeleton for all Hummingbot WS streams. + + Owns the while-True loop, subscriber check, heartbeat/error message + handling, permanent-error detection (``_is_permanent_ws_error``) and + exponential backoff capped at 60s. Each stream supplies only: + + - ``open_ws``: client -> the WS async context manager to enter + (e.g. ``lambda c: c.ws.market_data()``). + - ``subscribe``: performs the subscription on the open socket. + - ``on_message``: handles data messages (heartbeat/error are + handled here). + + ``backoff_on_empty_close`` reproduces the executor stream's + behavior: don't reset backoff on subscribe; after a clean close, + reset it only if at least one message arrived, otherwise back off + (guards against a server that accepts then immediately drops). + """ + from config_manager import get_config_manager + + cm = get_config_manager() + backoff = 5 + lower_label = label[0].lower() + label[1:] + subscribed_label = label if label.endswith("WS") else f"{label} WS" + + while True: + try: + client = await cm.get_client(server_name) + async with open_ws(client) as ws: + await subscribe(ws) + logger.info("%s subscribed: %s", subscribed_label, channel) + if not backoff_on_empty_close: + backoff = 5 + got_message = False + async for msg in ws: + if not self._has_subscribers(channel): + logger.info( + "No subscribers for %s, closing %s stream", + channel, + lower_label, + ) + return + + got_message = True + msg_type = msg.get("type") + if msg_type == "heartbeat": + continue + if msg_type == "error": + logger.warning( + "%s stream error for %s: %s", + label, + channel, + msg.get("message", "unknown error"), + ) + break + await on_message(msg) + + if backoff_on_empty_close: + # Connection closed cleanly — back off if short-lived + if got_message: + backoff = 5 + else: + logger.warning( + "%s stream closed immediately for %s, " + "reconnecting in %ds...", + label, + channel, + backoff, + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 60) + + except asyncio.CancelledError: + return + except Exception as e: + if self._is_permanent_ws_error(str(e)): + logger.warning( + "%s stream permanent error for %s: %s — giving up", + label, + channel, + e, + ) + return + + logger.warning( + "%s stream error for %s: %s, reconnecting in %ds...", + label, + channel, + e, + backoff, + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 60) + + # -- Trade streaming -- + + async def _trade_stream(self, channel: str) -> None: + parts = channel.split(":") + if len(parts) < 4: + return + _, server_name, connector, pair = parts[:4] + + async def subscribe(ws: Any) -> None: + await ws.subscribe_trades( + connector, + pair, + update_interval=1.0, + ) + + async def on_message(msg: dict) -> None: + if msg.get("type") != "trades": + return + trade_data = msg.get("data", []) + trades = [] + for t in trade_data: + if isinstance(t, dict): + trades.append( + { + "price": float(t.get("price", 0)), + "amount": float(t.get("amount", t.get("quantity", 0))), + "side": t.get("side", t.get("trade_type", "buy")).lower(), + "timestamp": float(t.get("timestamp", 0)), + } + ) + if trades: + await self.broadcast(channel, {"type": "trades", "data": trades}) + + await self._run_ws_stream( + channel, + server_name, + label="Trade", + open_ws=lambda client: client.ws.market_data(), + subscribe=subscribe, + on_message=on_message, + ) + + # -- Order book streaming -- + + async def _order_book_stream(self, channel: str) -> None: + parts = channel.split(":") + if len(parts) < 4: + return + _, server_name, connector, pair = parts[:4] + + async def subscribe(ws: Any) -> None: + await ws.subscribe_order_book( + connector, + pair, + depth=20, + update_interval=1.0, + ) + + async def on_message(msg: dict) -> None: + if msg.get("type") != "order_book": + return + raw_data = msg.get("data", {}) + bids = [] + asks = [] + for b in raw_data.get("bids") or []: + if isinstance(b, dict): + bids.append( + { + "price": float(b.get("price", 0)), + "amount": float(b.get("amount", b.get("quantity", 0))), + } + ) + elif isinstance(b, (list, tuple)) and len(b) >= 2: + bids.append({"price": float(b[0]), "amount": float(b[1])}) + for a in raw_data.get("asks") or []: + if isinstance(a, dict): + asks.append( + { + "price": float(a.get("price", 0)), + "amount": float(a.get("amount", a.get("quantity", 0))), + } + ) + elif isinstance(a, (list, tuple)) and len(a) >= 2: + asks.append({"price": float(a[0]), "amount": float(a[1])}) + await self.broadcast(channel, {"bids": bids, "asks": asks}) + + await self._run_ws_stream( + channel, + server_name, + label="Order book", + open_ws=lambda client: client.ws.market_data(), + subscribe=subscribe, + on_message=on_message, + ) + + # -- Executor streaming (via Hummingbot WS) -- + + async def _executor_stream(self, channel: str) -> None: + parts = channel.split(":") + if len(parts) < 2: + return + server_name = parts[1] + + from config_manager import get_config_manager + + cm = get_config_manager() + + # Try SDS cache first (pre-warmed by auto_subscribe_servers or REST prefetch) + from condor.server_data_service import ServerDataType, get_server_data_service + + if channel not in self._last_data: + sds = get_server_data_service() + cached = sds.get(server_name, ServerDataType.EXECUTORS) + if cached is not None: + executors = self._transform_executors(cached) + if executors: + await self.broadcast(channel, executors) + logger.info( + "Executor SDS cache hit: %d executors for %s", + len(executors), + channel, + ) + + # Wait briefly for SDS to be populated by a concurrent REST request + # (usePrefetchData fires getExecutors which calls get_or_fetch on SDS) + if channel not in self._last_data: + sds = get_server_data_service() + for _ in range(6): # up to 3 seconds + await asyncio.sleep(0.5) + cached = sds.get(server_name, ServerDataType.EXECUTORS) + if cached is not None: + executors = self._transform_executors(cached) + if executors: + await self.broadcast(channel, executors) + logger.info( + "Executor SDS cache populated during wait: %d executors for %s", + len(executors), + channel, + ) + break + + # Progressive pre-fetch only if we still have no data + if channel not in self._last_data: + try: + from condor.fetchers._pagination import walk_pages + from condor.fetchers.executors import ( + extract_executors_list as _extract_executors_list, + ) + + sds = get_server_data_service() + client = await cm.get_client(server_name) + all_raw: list[dict] = [] + # Rows already through ExecutorInfo, accumulated page by page. + # Transforming the *whole* accumulated list on every page made + # this quadratic: with FIRST_PAGE/NEXT_PAGE/MAX_PREFETCH below, + # a 5k history cost ~27k pydantic validations instead of 5k, all + # on the event loop the candle and bots broadcasts share. + # ``_transform_executors`` is a pure per-item map over a list + # (``extract_executors_list`` passes a list through untouched), + # so transforming each page once yields the identical snapshot. + transformed: list[dict] = [] + page_num = 0 + FIRST_PAGE = 50 + NEXT_PAGE = 500 + MAX_PREFETCH = 5000 + + # A small first page so the tab paints before the long pages land. + async for page in walk_pages( + client.executors.search_executors, + _extract_executors_list, + page_size=NEXT_PAGE, + first_page_size=FIRST_PAGE, + max_items=MAX_PREFETCH, + ): + all_raw.extend(page) + transformed.extend(self._transform_executors(page)) + + # Broadcast the accumulated snapshot after each page. It is + # copied because ``broadcast`` retains what it is handed as + # the channel's last-known payload, which the next page + # would otherwise keep mutating underneath it. + if transformed: + await self.broadcast(channel, list(transformed)) + logger.info( + "Executor pre-fetch page %d: %d executors (total %d) for %s", + page_num, + len(page), + len(transformed), + channel, + ) + page_num += 1 + + # Cache in SDS so other consumers benefit + sds.put(server_name, ServerDataType.EXECUTORS, all_raw) + except asyncio.CancelledError: + return + except Exception as e: + logger.warning("Executor pre-fetch failed for %s: %s", channel, e) + + async def subscribe(ws: Any) -> None: + await ws.subscribe_executors(update_interval=2.0) + + async def on_message(msg: dict) -> None: + if msg.get("type") != "executors": + return + executors = self._transform_executors(msg.get("data", [])) + await self._broadcast_update(channel, executors) + + await self._run_ws_stream( + channel, + server_name, + label="Executor", + open_ws=lambda client: client.ws.executors(), + subscribe=subscribe, + on_message=on_message, + backoff_on_empty_close=True, + ) + + # -- Bots WS streaming (via Hummingbot /ws/executors all_bots_status) -- + + async def _bots_ws_stream(self, channel: str) -> None: + """Stream all_bots_status from Hummingbot /ws/executors and update SDS cache.""" + parts = channel.split(":") + if len(parts) < 2: + return + server_name = parts[1] + + # Send SDS-cached bots data as initial snapshot + if channel not in self._last_data: + from condor.server_data_service import ( + ServerDataType, + get_server_data_service, + ) + + sds = get_server_data_service() + cached = sds.get(server_name, ServerDataType.BOTS_STATUS) + if cached is not None: + try: + data = self._transform_bots(cached) + await self.broadcast(channel, data) + except Exception as e: + logger.debug("Failed to send initial bots snapshot: %s", e) + + async def subscribe(ws: Any) -> None: + # all_bots_status is not in the client library, send raw + await ws._send( + { + "action": "subscribe", + "type": "all_bots_status", + "update_interval": 5.0, + } + ) + resp = await ws._receive() + if resp.get("type") == "error": + raise RuntimeError(f"Subscribe failed: {resp.get('message')}") + + async def on_message(msg: dict) -> None: + if msg.get("type") != "all_bots_status": + return + raw_data = msg.get("data", {}) + # Update SDS cache so REST and Telegram benefit + from condor.server_data_service import ( + ServerDataType, + get_server_data_service, + ) + + get_server_data_service().put( + server_name, ServerDataType.BOTS_STATUS, raw_data + ) + try: + data = self._transform_bots(raw_data) + self._overlay_stopping_state(server_name, data) + await self._broadcast_update(channel, data) + except Exception as e: + logger.debug("Failed to transform bots WS data: %s", e) + + await self._run_ws_stream( + channel, + server_name, + label="Bots WS", + open_ws=lambda client: client.ws.executors(), + subscribe=subscribe, + on_message=on_message, + ) + + # -- Positions WS streaming (via Hummingbot /ws/executors positions) -- + + async def _positions_ws_stream(self, channel: str) -> None: + """Stream positions from Hummingbot /ws/executors and update SDS cache.""" + parts = channel.split(":") + if len(parts) < 2: + return + server_name = parts[1] + + async def subscribe(ws: Any) -> None: + await ws.subscribe_positions(update_interval=5.0) + + async def on_message(msg: dict) -> None: + if msg.get("type") != "positions": + return + raw_data = msg.get("data", []) + # Update SDS cache + from condor.server_data_service import ( + ServerDataType, + get_server_data_service, + ) + + get_server_data_service().put( + server_name, ServerDataType.POSITIONS, raw_data + ) + await self._broadcast_update(channel, raw_data) + + await self._run_ws_stream( + channel, + server_name, + label="Positions WS", + open_ws=lambda client: client.ws.executors(), + subscribe=subscribe, + on_message=on_message, + ) + + # -- Performance WS streaming (via Hummingbot /ws/executors performance) -- + + async def _performance_ws_stream(self, channel: str) -> None: + """Stream performance from Hummingbot /ws/executors and update SDS cache.""" + parts = channel.split(":") + if len(parts) < 2: + return + server_name = parts[1] + + async def subscribe(ws: Any) -> None: + await ws.subscribe_performance(update_interval=5.0) + + async def on_message(msg: dict) -> None: + if msg.get("type") != "performance": + return + await self._broadcast_update(channel, msg.get("data", {})) + + await self._run_ws_stream( + channel, + server_name, + label="Performance WS", + open_ws=lambda client: client.ws.executors(), + subscribe=subscribe, + on_message=on_message, + ) + + # -- Controller Performance polling stream -- + + async def _controller_perf_stream(self, channel: str) -> None: + """Poll latest controller performance every 30s and broadcast snapshots.""" + parts = channel.split(":") + if len(parts) < 2: + return + server_name = parts[1] + + from config_manager import get_config_manager + + cm = get_config_manager() + backoff = 5 + + while True: + try: + if not self._has_subscribers(channel): + logger.info( + "No subscribers for %s, stopping controller perf stream", + channel, + ) + self._controller_perf_tasks.pop(channel, None) + return + + client = await cm.get_client(server_name) + result = ( + await client.bot_orchestration.get_latest_controller_performance() + ) + + # Normalize to list of snapshots + snapshots = [] + if isinstance(result, list): + snapshots = result + elif isinstance(result, dict): + data = result.get( + "data", result.get("snapshots", result.get("records", [])) + ) + if isinstance(data, list): + snapshots = data + elif isinstance(data, dict): + for key, val in data.items(): + if isinstance(val, dict): + val.setdefault("controller_id", key) + snapshots.append(val) + + if snapshots: + await self._broadcast_update(channel, {"snapshots": snapshots}) + backoff = 5 + + await asyncio.sleep(30) + + except asyncio.CancelledError: + return + except Exception as e: + logger.warning( + "Controller perf stream error for %s: %s, retrying in %ds...", + channel, + e, + backoff, + ) + await asyncio.sleep(backoff) + backoff = min(backoff * 2, 120) diff --git a/condor/web/ws_manager.py b/condor/web/ws_manager.py index 6122c5f37..5e886b317 100644 --- a/condor/web/ws_manager.py +++ b/condor/web/ws_manager.py @@ -1,18 +1,35 @@ +"""WebSocket connection registry and channel broadcaster for the dashboard. + +The upstream stream implementations live in ``condor.web.streams`` (candles in +``streams.candles``, the Hummingbot-bridged protocols in +``streams.hummingbot_ws``) as mixins over the small host surface this module +implements — see ``condor.web.streams.StreamHost``. ``WebSocketManager`` keeps +only connection/subscription bookkeeping, the SDS bridge, broadcasting and the +stream lifecycle registry. +""" + from __future__ import annotations import asyncio import json import logging -import math import time -from typing import TYPE_CHECKING, Any, Awaitable, Callable, Optional +from typing import TYPE_CHECKING, Any, Optional from fastapi import WebSocket -from condor import dex_candles +# Re-exported for back-compat: tests and callers import the candle helpers +# (and patch ``ws_manager.dex_candles``) through this module. +from condor import dex_candles # noqa: F401 from condor.asyncutil import TaskSet -from condor.fetchers.market_data import fetch_historical_candles, normalize_candle from condor.web.auth import decode_jwt +from condor.web.streams.candles import ( # noqa: F401 + CandleStreamsMixin, + _CandleBuffer, + _coerce_duration, + parse_candle_channel, +) +from condor.web.streams.hummingbot_ws import HummingbotStreamsMixin if TYPE_CHECKING: from condor.server_data_service import CacheKey @@ -73,135 +90,6 @@ def channel_for_key(key: CacheKey) -> str | None: return ":".join([prefix, key.server, *segments]) -# Interval string -> seconds for buffer sizing -_INTERVAL_SECONDS: dict[str, int] = { - "1s": 1, - "5s": 5, - "15s": 15, - "30s": 30, - "1m": 60, - "3m": 180, - "5m": 300, - "15m": 900, - "30m": 1800, - "1h": 3600, - "2h": 7200, - "4h": 14400, - "1d": 86400, - "1w": 604800, -} - -# Auto-cleanup candle buffers unused for this long -_CANDLE_BUFFER_IDLE_TTL = 600 # 10 minutes - - -def _coerce_duration(value: object) -> int | None: - """Coerce a client-supplied candle duration to a positive int (seconds). - - WS messages are untrusted input: a non-numeric value must not raise, or the - ValueError would escape handle_message and tear down the client's entire - multiplexed connection. Returns None for missing/invalid/non-positive values. - """ - if value is None or isinstance(value, bool): - return None - try: - duration = int(float(value)) - except (TypeError, ValueError): - logger.warning("Ignoring invalid candle duration from client: %r", value) - return None - return duration if duration > 0 else None - - -def parse_candle_channel(channel: str) -> tuple[str, str, str, str, str | None] | None: - """Split a candle channel into its parts, or ``None`` if it is malformed. - - The channel is ``candles:{server}:{connector}:{pair}:{interval}[:{pool_address}]``. - The pool address is an **optional sixth segment**, set by the DEX page so a - chart is pinned to the exact pool the user opened instead of whichever pool - ``dex_candles._resolve_pool`` judges deepest for the pair. A five-part channel - (every CLOB chart) parses to ``pool_address=None`` and is byte-identical to - what it was before the segment existed. - - Every consumer must go through here: unpacking five names off ``split(":")`` - raises ``ValueError`` on a six-part channel, and each of the three call sites - fails silently in a different way (a stream that never backfills, a backfill - that never ticks). - """ - parts = channel.split(":") - if len(parts) < 5: - return None - server_name, connector, pair, interval = parts[1:5] - pool_address = parts[5] if len(parts) > 5 else None - return server_name, connector, pair, interval, pool_address or None - - -class _CandleBuffer: - """Per-channel candle buffer with dynamic sizing based on interval + duration.""" - - __slots__ = ("interval", "_data", "_max_size", "last_accessed") - - def __init__(self, interval: str, duration_seconds: int = 3600): - self.interval = interval - self._data: dict[float, dict] = {} - self._max_size: int = 200 - self.last_accessed: float = time.monotonic() - self.set_duration(duration_seconds) - - def set_duration(self, duration_seconds: int) -> int: - """Resize buffer for the given duration. Returns the new max size.""" - interval_sec = _INTERVAL_SECONDS.get(self.interval, 60) - needed = math.ceil(duration_seconds / interval_sec) - new_max = max(needed, 200) # minimum 200 - old_max = self._max_size - self._max_size = new_max - self.last_accessed = time.monotonic() - self._evict() - if new_max != old_max: - logger.debug( - "Candle buffer resized %s: %d -> %d (duration=%ds)", - self.interval, - old_max, - new_max, - duration_seconds, - ) - return new_max - - def upsert(self, candle: dict) -> None: - self._data[candle["timestamp"]] = candle - self._evict() - - def upsert_many(self, candles: list[dict]) -> None: - for c in candles: - self._data[c["timestamp"]] = c - self._evict() - - def get_sorted(self) -> list[dict]: - self.last_accessed = time.monotonic() - return sorted(self._data.values(), key=lambda c: c["timestamp"]) - - def _evict(self) -> None: - excess = len(self._data) - self._max_size - if excess <= 0: - return - # Drop the `excess` oldest timestamps in a single O(n log n) pass - # instead of calling min() (O(n)) per evicted candle. - for ts in sorted(self._data)[:excess]: - del self._data[ts] - - @property - def size(self) -> int: - return len(self._data) - - @property - def max_size(self) -> int: - return self._max_size - - @property - def needs_backfill(self) -> bool: - """True if buffer has room for significantly more candles.""" - return len(self._data) < self._max_size * 0.5 - - class _Connection: __slots__ = ("ws", "user_id", "channels") @@ -211,16 +99,17 @@ def __init__(self, ws: WebSocket, user_id: int): self.channels: set[str] = set() -class WebSocketManager: +class WebSocketManager(CandleStreamsMixin, HummingbotStreamsMixin): """Manages WebSocket connections and channel-based data broadcasting. Subscribes to ServerDataService for data updates and broadcasts - to connected WebSocket clients. Candle streaming uses dedicated - WebSocket connections with dynamic per-channel buffering. + to connected WebSocket clients. The upstream stream protocols are + supplied by the mixins (``CandleStreamsMixin``, + ``HummingbotStreamsMixin``); this class owns their shared state and + the host surface they run on (``broadcast``, ``_send``, + ``_has_subscribers``, ``_ensure_stream``, ``_oneshot_tasks``). """ - _CANDLE_KEEP_ALIVE = 300 # 5-minute grace period before tearing down candle streams - def __init__(self): self._connections: list[_Connection] = [] self._last_data: dict[str, Any] = {} # channel -> last broadcast payload @@ -264,11 +153,6 @@ def _server_from_channel(channel: str) -> str | None: parts = channel.split(":") return parts[1] if len(parts) >= 2 and parts[1] else None - @staticmethod - def _normalize_candle(c: Any) -> dict | None: - """Normalize a candle from any format to a uniform dict with float values.""" - return normalize_candle(c) - # -- Lifecycle -- def start(self) -> None: @@ -450,135 +334,7 @@ async def handle_message(self, conn: _Connection, raw: str) -> None: if channel.startswith("candles:") and duration: await self._handle_candle_duration_change(conn, channel, duration) - async def _handle_candle_subscribe( - self, conn: _Connection, channel: str, duration: object - ) -> None: - """Handle candle channel subscription with optional duration.""" - parts = channel.split(":") - if len(parts) < 5: - return - interval = parts[4] - dur = _coerce_duration(duration) or 3 * 86400 # default 3 days - - # Cancel any pending teardown — a subscriber just came back - timer = self._candle_teardown_timers.pop(channel, None) - if timer is not None: - timer.cancel() - logger.debug("Cancelled pending candle teardown for %s", channel) - - buf = self._candle_buffers.get(channel) - if buf is None: - buf = _CandleBuffer(interval, dur) - self._candle_buffers[channel] = buf - else: - # Expand buffer if this client needs more - if dur > 0: - old_max = buf.max_size - buf.set_duration(dur) - if buf.max_size > old_max and buf.needs_backfill: - self._oneshot_tasks.track( - asyncio.create_task( - self._backfill_candles(channel), - name=f"backfill:{channel}", - ) - ) - - # Send buffered candles as initial snapshot - sorted_candles = buf.get_sorted() - if sorted_candles: - await self._send(conn, channel, {"type": "candles", "data": sorted_candles}) - - # If the stream task is still running (kept alive during grace period), skip restart - self._ensure_stream("candles", channel) - - async def _handle_candle_duration_change( - self, conn: _Connection, channel: str, duration: int - ) -> None: - """Handle duration change for an existing candle subscription. - - Only grows the buffer — never shrinks. The frontend manages its own - display window; the backend just ensures enough history is buffered. - """ - buf = self._candle_buffers.get(channel) - if buf is None: - return - old_max = buf.max_size - # Only expand — skip if requested duration would shrink the buffer - interval_sec = _INTERVAL_SECONDS.get(buf.interval, 60) - needed = max(math.ceil(duration / interval_sec), 200) - if needed <= old_max: - return - buf.set_duration(duration) - if buf.needs_backfill: - await self._backfill_candles(channel) - # Broadcast updated snapshot to ALL subscribers on this channel - sorted_candles = buf.get_sorted() - if sorted_candles: - await self.broadcast(channel, {"type": "candles", "data": sorted_candles}) - - async def _backfill_candles(self, channel: str) -> None: - """Fetch historical candles to fill the buffer gap.""" - parsed = parse_candle_channel(channel) - if parsed is None: - return - server_name, connector, pair, interval, pool_address = parsed - buf = self._candle_buffers.get(channel) - if buf is None: - return - - from config_manager import get_config_manager - - cm = get_config_manager() - try: - interval_sec = _INTERVAL_SECONDS.get(interval, 60) - end_time = int(time.time()) - start_time = end_time - (buf.max_size * interval_sec) - - logger.info( - "Backfilling candles for %s: need %d, have %d, fetching %ds-%ds", - channel, - buf.max_size, - buf.size, - start_time, - end_time, - ) - - if dex_candles.uses_gecko_candles(connector): - candles = await dex_candles.fetch_dex_candles( - connector, pool_address, pair, interval, start_time, end_time - ) - if candles: - buf.upsert_many(candles) - logger.info( - "Backfilled %d GeckoTerminal candles for %s (buffer: %d/%d)", - len(candles), - channel, - buf.size, - buf.max_size, - ) - return - - client = await cm.get_client(server_name) - candles = await fetch_historical_candles( - client, - connector, - pair, - interval, - start_time=start_time, - end_time=end_time, - limit=min(buf.max_size, 5000), - ) - if candles: - buf.upsert_many(candles) - logger.info( - "Backfilled %d candles for %s (buffer: %d/%d)", - len(candles), - channel, - buf.size, - buf.max_size, - ) - except Exception as e: - logger.warning("Candle backfill failed for %s: %s", channel, e) + # -- SDS bridge -- async def _subscribe_sds(self, channel: str) -> None: """Subscribe to SDS for a channel and prime cache.""" @@ -699,36 +455,6 @@ def _unsubscribe_portfolio_history(self, server_name: str) -> None: # -- SDS listener -- - @staticmethod - def _transform_executors(raw_data: Any) -> list[dict]: - """Transform raw executor data to ExecutorInfo-compatible dicts for WS broadcast.""" - from condor.fetchers.executors import extract_executors_list - from condor.web.models import ExecutorInfo - - executors_list = extract_executors_list(raw_data) - result = [] - for ex in executors_list: - info = ExecutorInfo.from_raw(ex) - if info: - result.append(info.model_dump()) - return result - - @staticmethod - def _transform_bots(raw_data: Any) -> dict: - """Transform raw BOTS_STATUS data to BotsPageResponse-compatible dict for WS broadcast.""" - from condor.fetchers.bots import build_bots_page - - return build_bots_page(raw_data) - - @staticmethod - def _overlay_stopping_state(server_name: str, data: dict) -> None: - """Apply transitional 'stopping' state to WS broadcast data.""" - from condor.web.routes.bots import overlay_stopping_state - - overlay_stopping_state( - server_name, data.get("controllers", []), data.get("bots", []) - ) - def _on_data_update(self, key: CacheKey, value: Any) -> None: """Called by SDS on every cache write. Maps the key to a WS channel and broadcasts. ``key`` is an SDS ``CacheKey``: server, data type and params.""" @@ -891,913 +617,6 @@ def _maybe_stop_stream(self, prefix: str, channel: str) -> None: logger.info(spec["stop_log"], channel) self._last_data.pop(channel, None) - def _maybe_stop_candle_stream(self, channel: str) -> None: - # If subscribers still exist, cancel any pending teardown and return - if self._has_subscribers(channel): - timer = self._candle_teardown_timers.pop(channel, None) - if timer is not None: - timer.cancel() - return - - # Already have a pending teardown scheduled — nothing to do - if channel in self._candle_teardown_timers: - return - - # Schedule deferred teardown - loop = asyncio.get_event_loop() - handle = loop.call_later( - self._CANDLE_KEEP_ALIVE, - self._deferred_stop_candle_stream, - channel, - ) - self._candle_teardown_timers[channel] = handle - logger.info( - "Scheduled candle stream teardown for %s in %ds", - channel, - self._CANDLE_KEEP_ALIVE, - ) - - def _deferred_stop_candle_stream(self, channel: str) -> None: - """Actually tear down a candle stream after the grace period.""" - self._candle_teardown_timers.pop(channel, None) - - # Re-check: subscribers may have appeared during the grace period - if self._has_subscribers(channel): - logger.debug( - "Candle teardown cancelled — subscribers returned for %s", channel - ) - return - - task = self._candle_tasks.pop(channel, None) - if task and not task.done(): - task.cancel() - logger.info("Deferred candle stream teardown completed for %s", channel) - poll_task = self._candle_poll_tasks.pop(channel, None) - if poll_task and not poll_task.done(): - poll_task.cancel() - self._last_candle_ws_update.pop(channel, None) - self._candle_first_msg_logged.discard(channel) - # Snapshot-on-resubscribe is served by _candle_buffers, not _last_data - self._last_data.pop(channel, None) - # NOTE: candle buffer is NOT deleted — the existing idle cleanup loop handles that - - @staticmethod - def _is_permanent_ws_error(error_str: str) -> bool: - """True if a stream error will never succeed on retry. - - Besides auth/not-found (401/403/404), an invalid trading pair (wrong - symbol for this connector) will never become valid, so retrying - forever just spams logs and hammers the exchange until Condor - restarts (issue #134). - """ - lowered = error_str.lower() - return ( - any(code in error_str for code in ("401", "403", "404")) - or "appears to be invalid" in lowered - or "invalid symbol" in lowered - ) - - async def _run_ws_stream( - self, - channel: str, - server_name: str, - *, - label: str, - open_ws: Callable[[Any], Any], - subscribe: Callable[[Any], Awaitable[None]], - on_message: Callable[[dict], Awaitable[None]], - backoff_on_empty_close: bool = False, - ) -> None: - """Shared connect/reconnect skeleton for all Hummingbot WS streams. - - Owns the while-True loop, subscriber check, heartbeat/error message - handling, permanent-error detection (``_is_permanent_ws_error``) and - exponential backoff capped at 60s. Each stream supplies only: - - - ``open_ws``: client -> the WS async context manager to enter - (e.g. ``lambda c: c.ws.market_data()``). - - ``subscribe``: performs the subscription on the open socket. - - ``on_message``: handles data messages (heartbeat/error are - handled here). - - ``backoff_on_empty_close`` reproduces the executor stream's - behavior: don't reset backoff on subscribe; after a clean close, - reset it only if at least one message arrived, otherwise back off - (guards against a server that accepts then immediately drops). - """ - from config_manager import get_config_manager - - cm = get_config_manager() - backoff = 5 - lower_label = label[0].lower() + label[1:] - subscribed_label = label if label.endswith("WS") else f"{label} WS" - - while True: - try: - client = await cm.get_client(server_name) - async with open_ws(client) as ws: - await subscribe(ws) - logger.info("%s subscribed: %s", subscribed_label, channel) - if not backoff_on_empty_close: - backoff = 5 - got_message = False - async for msg in ws: - if not self._has_subscribers(channel): - logger.info( - "No subscribers for %s, closing %s stream", - channel, - lower_label, - ) - return - - got_message = True - msg_type = msg.get("type") - if msg_type == "heartbeat": - continue - if msg_type == "error": - logger.warning( - "%s stream error for %s: %s", - label, - channel, - msg.get("message", "unknown error"), - ) - break - await on_message(msg) - - if backoff_on_empty_close: - # Connection closed cleanly — back off if short-lived - if got_message: - backoff = 5 - else: - logger.warning( - "%s stream closed immediately for %s, " - "reconnecting in %ds...", - label, - channel, - backoff, - ) - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 60) - - except asyncio.CancelledError: - return - except Exception as e: - if self._is_permanent_ws_error(str(e)): - logger.warning( - "%s stream permanent error for %s: %s — giving up", - label, - channel, - e, - ) - return - - logger.warning( - "%s stream error for %s: %s, reconnecting in %ds...", - label, - channel, - e, - backoff, - ) - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 60) - - async def _candle_stream(self, channel: str) -> None: - parsed = parse_candle_channel(channel) - if parsed is None: - return - server_name, connector, pair, interval, pool_address = parsed - - from config_manager import get_config_manager - - cm = get_config_manager() - backoff = 5 - - # Start REST poll fallback alongside the WS stream - self._ensure_candle_poll_fallback(channel) - - if dex_candles.uses_gecko_candles(connector): - # No CandlesFactory feed to subscribe to (xrpl, DEX networks): the - # poll loop above is the whole stream. Subscribing anyway would just - # reconnect-loop on an error the API can never stop returning. - logger.info( - "Candle stream for %s is GeckoTerminal-polled (no API feed)", channel - ) - return - - while True: - try: - client = await cm.get_client(server_name) - async with client.ws.market_data() as ws: - await ws.subscribe_candles( - connector, - pair, - interval=interval, - max_records=100, - update_interval=1.0, - ) - logger.info("Candle WS subscribed: %s", channel) - backoff = 5 # Reset on successful connection - async for msg in ws: - # Log first message per channel at INFO for diagnostics - if channel not in self._candle_first_msg_logged: - self._candle_first_msg_logged.add(channel) - logger.info( - "Candle WS first message for %s: %s", - channel, - ( - json.dumps(msg)[:500] - if isinstance(msg, dict) - else type(msg).__name__ - ), - ) - else: - logger.debug( - "Candle WS raw msg keys for %s: %s", - channel, - ( - list(msg.keys()) - if isinstance(msg, dict) - else type(msg).__name__ - ), - ) - - msg_type = msg.get("type") - if msg_type == "candle_update": - raw = msg.get("data") - candle = self._normalize_candle(raw) if raw else None - if candle: - self._last_candle_ws_update[channel] = time.monotonic() - self._upsert_candle_buffer(channel, candle) - await self.broadcast( - channel, - {"type": "candle_update", "candle": candle}, - ) - elif msg_type == "candles": - raw_list = msg.get("data") or [] - candles = [ - c - for r in raw_list - if (c := self._normalize_candle(r)) is not None - ] - if candles: - self._last_candle_ws_update[channel] = time.monotonic() - self._upsert_candle_buffer_many(channel, candles) - await self.broadcast( - channel, - {"type": "candles", "data": candles}, - ) - elif msg_type in ("heartbeat", "subscribed"): - continue - elif msg_type == "error": - error_msg = msg.get("message", "unknown error") - logger.warning( - "Candle stream error for %s: %s — continuing", - channel, - error_msg, - ) - await self.broadcast( - channel, - { - "type": "error", - "message": f"Stream error: {error_msg}", - }, - ) - # Don't break — the WS may still be alive. - # If truly dead, next recv raises and we reconnect. - continue - else: - logger.info( - "Candle stream unrecognized msg type for %s: type=%s keys=%s", - channel, - msg_type, - ( - list(msg.keys()) - if isinstance(msg, dict) - else type(msg).__name__ - ), - ) - - except asyncio.CancelledError: - return - except Exception as e: - error_str = str(e) - # Detect permanent failures — don't retry (issue #134, see - # _is_permanent_ws_error). - if self._is_permanent_ws_error(error_str): - logger.warning( - "Candle stream permanent error for %s: %s — giving up", - channel, - e, - ) - await self.broadcast( - channel, - {"type": "error", "message": f"Stream failed: {error_str}"}, - ) - # Stop the paired REST poll fallback — nothing valid to poll. - poll_task = self._candle_poll_tasks.pop(channel, None) - if poll_task and not poll_task.done(): - poll_task.cancel() - return - - logger.warning( - "Candle stream error for %s: %s, reconnecting in %ds...", - channel, - e, - backoff, - ) - await self.broadcast( - channel, - { - "type": "error", - "message": f"Connection lost, retrying in {backoff}s", - }, - ) - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 60) - - # -- Candle REST poll fallback -- - - def _ensure_candle_poll_fallback(self, channel: str) -> None: - """Start a REST poll fallback task for a candle channel if not already running.""" - if ( - channel in self._candle_poll_tasks - and not self._candle_poll_tasks[channel].done() - ): - return - self._candle_poll_tasks[channel] = asyncio.create_task( - self._candle_poll_fallback(channel) - ) - - async def _candle_poll_fallback(self, channel: str) -> None: - """Periodically poll REST for candles when WS stream goes silent. - - For a GeckoTerminal-backed connector there is no WS stream to fall back - from — this loop *is* the feed, so it polls from the first tick and never - waits for staleness. - """ - parsed = parse_candle_channel(channel) - if parsed is None: - return - server_name, connector, pair, interval, pool_address = parsed - interval_sec = _INTERVAL_SECONDS.get(interval, 60) - gecko = dex_candles.uses_gecko_candles(connector) - # How long without a WS update before we consider it stale - stale_threshold = max(interval_sec, 15) - # How often to poll REST once stale (keep the last candle fresh). Gecko - # polls stay well clear of its per-minute rate limit — every chart on the - # dashboard shares one budget — while still refreshing a forming candle. - poll_interval = ( - max(min(interval_sec // 2, 60), 30) if gecko else min(interval_sec, 10) - ) - was_stale = gecko - - from config_manager import get_config_manager - - try: - while True: - await asyncio.sleep(poll_interval if was_stale else stale_threshold) - - last_update = self._last_candle_ws_update.get(channel) - if ( - last_update is not None - and (time.monotonic() - last_update) <= stale_threshold - ): - if was_stale: - logger.info( - "Candle WS stream resumed for %s, stopping REST fallback polling", - channel, - ) - was_stale = False - continue - - # Stream is stale — poll REST - if not was_stale: - logger.warning( - "Candle WS stream appears stale for %s (threshold=%ds), polling REST fallback", - channel, - stale_threshold, - ) - was_stale = True - - try: - now = int(time.time()) - if gecko: - candles = await dex_candles.fetch_dex_candles( - connector, - pool_address, - pair, - interval, - now - interval_sec * 5, - now, - # The shared OHLCV cache holds a pool for minutes, - # long enough to freeze a live chart. - use_cache=False, - ) - else: - cm = get_config_manager() - client = await cm.get_client(server_name) - # No `limit`: this poll only wants the fresh tail of the - # live range, so an empty historical answer stays empty - # rather than pulling a full unrelated window. - candles = await fetch_historical_candles( - client, - connector, - pair, - interval, - start_time=now - interval_sec * 5, - end_time=now, - ) - if candles: - buf = self._candle_buffers.get(channel) - # Broadcast if we have newer candles OR if the latest - # candle's OHLCV changed (same timestamp, updated values) - newest_buf_ts = ( - max(buf._data.keys()) if buf and buf._data else 0 - ) - newest_poll_ts = max(c["timestamp"] for c in candles) - changed = newest_poll_ts > newest_buf_ts - if not changed and buf and newest_poll_ts in buf._data: - # Same timestamp — check if OHLCV actually changed - old = buf._data[newest_poll_ts] - new = next( - c for c in candles if c["timestamp"] == newest_poll_ts - ) - changed = any( - old.get(k) != new.get(k) - for k in ("open", "high", "low", "close", "volume") - ) - if changed: - self._upsert_candle_buffer_many(channel, candles) - await self.broadcast( - channel, - {"type": "candles", "data": candles}, - ) - logger.debug( - "REST fallback delivered %d candles for %s", - len(candles), - channel, - ) - except Exception as e: - logger.debug("REST candle poll failed for %s: %s", channel, e) - - except asyncio.CancelledError: - return - - # -- Trade streaming -- - - async def _trade_stream(self, channel: str) -> None: - parts = channel.split(":") - if len(parts) < 4: - return - _, server_name, connector, pair = parts[:4] - - async def subscribe(ws: Any) -> None: - await ws.subscribe_trades( - connector, - pair, - update_interval=1.0, - ) - - async def on_message(msg: dict) -> None: - if msg.get("type") != "trades": - return - trade_data = msg.get("data", []) - trades = [] - for t in trade_data: - if isinstance(t, dict): - trades.append( - { - "price": float(t.get("price", 0)), - "amount": float(t.get("amount", t.get("quantity", 0))), - "side": t.get("side", t.get("trade_type", "buy")).lower(), - "timestamp": float(t.get("timestamp", 0)), - } - ) - if trades: - await self.broadcast(channel, {"type": "trades", "data": trades}) - - await self._run_ws_stream( - channel, - server_name, - label="Trade", - open_ws=lambda client: client.ws.market_data(), - subscribe=subscribe, - on_message=on_message, - ) - - # -- Candle buffer helpers -- - - def _get_or_create_candle_buffer(self, channel: str) -> _CandleBuffer: - """Get existing buffer or create with default duration.""" - buf = self._candle_buffers.get(channel) - if buf is None: - parts = channel.split(":") - interval = parts[4] if len(parts) >= 5 else "1m" - buf = _CandleBuffer(interval, 3 * 86400) # default 3 days - self._candle_buffers[channel] = buf - return buf - - def _upsert_candle_buffer(self, channel: str, candle: dict) -> None: - """Upsert a single candle into the per-channel buffer.""" - buf = self._get_or_create_candle_buffer(channel) - buf.upsert(candle) - - def _upsert_candle_buffer_many(self, channel: str, candles: list[dict]) -> None: - """Upsert multiple candles into the per-channel buffer.""" - buf = self._get_or_create_candle_buffer(channel) - buf.upsert_many(candles) - - async def _candle_buffer_cleanup_loop(self) -> None: - """Periodically remove candle buffers that haven't been accessed.""" - try: - while True: - await asyncio.sleep(60) # check every minute - now = time.monotonic() - stale = [ - ch - for ch, buf in self._candle_buffers.items() - if (now - buf.last_accessed) > _CANDLE_BUFFER_IDLE_TTL - and not any(ch in c.channels for c in self._connections) - ] - for ch in stale: - buf = self._candle_buffers.pop(ch, None) - if buf: - logger.info( - "Cleaned up idle candle buffer: %s (%d candles)", - ch, - buf.size, - ) - except asyncio.CancelledError: - return - - # -- Order book streaming -- - - async def _order_book_stream(self, channel: str) -> None: - parts = channel.split(":") - if len(parts) < 4: - return - _, server_name, connector, pair = parts[:4] - - async def subscribe(ws: Any) -> None: - await ws.subscribe_order_book( - connector, - pair, - depth=20, - update_interval=1.0, - ) - - async def on_message(msg: dict) -> None: - if msg.get("type") != "order_book": - return - raw_data = msg.get("data", {}) - bids = [] - asks = [] - for b in raw_data.get("bids") or []: - if isinstance(b, dict): - bids.append( - { - "price": float(b.get("price", 0)), - "amount": float(b.get("amount", b.get("quantity", 0))), - } - ) - elif isinstance(b, (list, tuple)) and len(b) >= 2: - bids.append({"price": float(b[0]), "amount": float(b[1])}) - for a in raw_data.get("asks") or []: - if isinstance(a, dict): - asks.append( - { - "price": float(a.get("price", 0)), - "amount": float(a.get("amount", a.get("quantity", 0))), - } - ) - elif isinstance(a, (list, tuple)) and len(a) >= 2: - asks.append({"price": float(a[0]), "amount": float(a[1])}) - await self.broadcast(channel, {"bids": bids, "asks": asks}) - - await self._run_ws_stream( - channel, - server_name, - label="Order book", - open_ws=lambda client: client.ws.market_data(), - subscribe=subscribe, - on_message=on_message, - ) - - # -- Executor streaming (via Hummingbot WS) -- - - async def _executor_stream(self, channel: str) -> None: - parts = channel.split(":") - if len(parts) < 2: - return - server_name = parts[1] - - from config_manager import get_config_manager - - cm = get_config_manager() - - # Try SDS cache first (pre-warmed by auto_subscribe_servers or REST prefetch) - from condor.server_data_service import ServerDataType, get_server_data_service - - if channel not in self._last_data: - sds = get_server_data_service() - cached = sds.get(server_name, ServerDataType.EXECUTORS) - if cached is not None: - executors = self._transform_executors(cached) - if executors: - await self.broadcast(channel, executors) - logger.info( - "Executor SDS cache hit: %d executors for %s", - len(executors), - channel, - ) - - # Wait briefly for SDS to be populated by a concurrent REST request - # (usePrefetchData fires getExecutors which calls get_or_fetch on SDS) - if channel not in self._last_data: - sds = get_server_data_service() - for _ in range(6): # up to 3 seconds - await asyncio.sleep(0.5) - cached = sds.get(server_name, ServerDataType.EXECUTORS) - if cached is not None: - executors = self._transform_executors(cached) - if executors: - await self.broadcast(channel, executors) - logger.info( - "Executor SDS cache populated during wait: %d executors for %s", - len(executors), - channel, - ) - break - - # Progressive pre-fetch only if we still have no data - if channel not in self._last_data: - try: - from condor.fetchers._pagination import walk_pages - from condor.fetchers.executors import ( - extract_executors_list as _extract_executors_list, - ) - - sds = get_server_data_service() - client = await cm.get_client(server_name) - all_raw: list[dict] = [] - # Rows already through ExecutorInfo, accumulated page by page. - # Transforming the *whole* accumulated list on every page made - # this quadratic: with FIRST_PAGE/NEXT_PAGE/MAX_PREFETCH below, - # a 5k history cost ~27k pydantic validations instead of 5k, all - # on the event loop the candle and bots broadcasts share. - # ``_transform_executors`` is a pure per-item map over a list - # (``extract_executors_list`` passes a list through untouched), - # so transforming each page once yields the identical snapshot. - transformed: list[dict] = [] - page_num = 0 - FIRST_PAGE = 50 - NEXT_PAGE = 500 - MAX_PREFETCH = 5000 - - # A small first page so the tab paints before the long pages land. - async for page in walk_pages( - client.executors.search_executors, - _extract_executors_list, - page_size=NEXT_PAGE, - first_page_size=FIRST_PAGE, - max_items=MAX_PREFETCH, - ): - all_raw.extend(page) - transformed.extend(self._transform_executors(page)) - - # Broadcast the accumulated snapshot after each page. It is - # copied because ``broadcast`` retains what it is handed as - # the channel's last-known payload, which the next page - # would otherwise keep mutating underneath it. - if transformed: - await self.broadcast(channel, list(transformed)) - logger.info( - "Executor pre-fetch page %d: %d executors (total %d) for %s", - page_num, - len(page), - len(transformed), - channel, - ) - page_num += 1 - - # Cache in SDS so other consumers benefit - sds.put(server_name, ServerDataType.EXECUTORS, all_raw) - except asyncio.CancelledError: - return - except Exception as e: - logger.warning("Executor pre-fetch failed for %s: %s", channel, e) - - async def subscribe(ws: Any) -> None: - await ws.subscribe_executors(update_interval=2.0) - - async def on_message(msg: dict) -> None: - if msg.get("type") != "executors": - return - executors = self._transform_executors(msg.get("data", [])) - await self._broadcast_update(channel, executors) - - await self._run_ws_stream( - channel, - server_name, - label="Executor", - open_ws=lambda client: client.ws.executors(), - subscribe=subscribe, - on_message=on_message, - backoff_on_empty_close=True, - ) - - # -- Bots WS streaming (via Hummingbot /ws/executors all_bots_status) -- - - async def _bots_ws_stream(self, channel: str) -> None: - """Stream all_bots_status from Hummingbot /ws/executors and update SDS cache.""" - parts = channel.split(":") - if len(parts) < 2: - return - server_name = parts[1] - - # Send SDS-cached bots data as initial snapshot - if channel not in self._last_data: - from condor.server_data_service import ( - ServerDataType, - get_server_data_service, - ) - - sds = get_server_data_service() - cached = sds.get(server_name, ServerDataType.BOTS_STATUS) - if cached is not None: - try: - data = self._transform_bots(cached) - await self.broadcast(channel, data) - except Exception as e: - logger.debug("Failed to send initial bots snapshot: %s", e) - - async def subscribe(ws: Any) -> None: - # all_bots_status is not in the client library, send raw - await ws._send( - { - "action": "subscribe", - "type": "all_bots_status", - "update_interval": 5.0, - } - ) - resp = await ws._receive() - if resp.get("type") == "error": - raise RuntimeError(f"Subscribe failed: {resp.get('message')}") - - async def on_message(msg: dict) -> None: - if msg.get("type") != "all_bots_status": - return - raw_data = msg.get("data", {}) - # Update SDS cache so REST and Telegram benefit - from condor.server_data_service import ( - ServerDataType, - get_server_data_service, - ) - - get_server_data_service().put( - server_name, ServerDataType.BOTS_STATUS, raw_data - ) - try: - data = self._transform_bots(raw_data) - self._overlay_stopping_state(server_name, data) - await self._broadcast_update(channel, data) - except Exception as e: - logger.debug("Failed to transform bots WS data: %s", e) - - await self._run_ws_stream( - channel, - server_name, - label="Bots WS", - open_ws=lambda client: client.ws.executors(), - subscribe=subscribe, - on_message=on_message, - ) - - # -- Positions WS streaming (via Hummingbot /ws/executors positions) -- - - async def _positions_ws_stream(self, channel: str) -> None: - """Stream positions from Hummingbot /ws/executors and update SDS cache.""" - parts = channel.split(":") - if len(parts) < 2: - return - server_name = parts[1] - - async def subscribe(ws: Any) -> None: - await ws.subscribe_positions(update_interval=5.0) - - async def on_message(msg: dict) -> None: - if msg.get("type") != "positions": - return - raw_data = msg.get("data", []) - # Update SDS cache - from condor.server_data_service import ( - ServerDataType, - get_server_data_service, - ) - - get_server_data_service().put( - server_name, ServerDataType.POSITIONS, raw_data - ) - await self._broadcast_update(channel, raw_data) - - await self._run_ws_stream( - channel, - server_name, - label="Positions WS", - open_ws=lambda client: client.ws.executors(), - subscribe=subscribe, - on_message=on_message, - ) - - # -- Performance WS streaming (via Hummingbot /ws/executors performance) -- - - async def _performance_ws_stream(self, channel: str) -> None: - """Stream performance from Hummingbot /ws/executors and update SDS cache.""" - parts = channel.split(":") - if len(parts) < 2: - return - server_name = parts[1] - - async def subscribe(ws: Any) -> None: - await ws.subscribe_performance(update_interval=5.0) - - async def on_message(msg: dict) -> None: - if msg.get("type") != "performance": - return - await self._broadcast_update(channel, msg.get("data", {})) - - await self._run_ws_stream( - channel, - server_name, - label="Performance WS", - open_ws=lambda client: client.ws.executors(), - subscribe=subscribe, - on_message=on_message, - ) - - # -- Controller Performance polling stream -- - - async def _controller_perf_stream(self, channel: str) -> None: - """Poll latest controller performance every 30s and broadcast snapshots.""" - parts = channel.split(":") - if len(parts) < 2: - return - server_name = parts[1] - - from config_manager import get_config_manager - - cm = get_config_manager() - backoff = 5 - - while True: - try: - if not self._has_subscribers(channel): - logger.info( - "No subscribers for %s, stopping controller perf stream", - channel, - ) - self._controller_perf_tasks.pop(channel, None) - return - - client = await cm.get_client(server_name) - result = ( - await client.bot_orchestration.get_latest_controller_performance() - ) - - # Normalize to list of snapshots - snapshots = [] - if isinstance(result, list): - snapshots = result - elif isinstance(result, dict): - data = result.get( - "data", result.get("snapshots", result.get("records", [])) - ) - if isinstance(data, list): - snapshots = data - elif isinstance(data, dict): - for key, val in data.items(): - if isinstance(val, dict): - val.setdefault("controller_id", key) - snapshots.append(val) - - if snapshots: - await self._broadcast_update(channel, {"snapshots": snapshots}) - backoff = 5 - - await asyncio.sleep(30) - - except asyncio.CancelledError: - return - except Exception as e: - logger.warning( - "Controller perf stream error for %s: %s, retrying in %ds...", - channel, - e, - backoff, - ) - await asyncio.sleep(backoff) - backoff = min(backoff * 2, 120) - # -- Singleton -- From 1cbc5dcb357c72b8d988ce6e2547a87adea5c2df Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 23:21:15 +0300 Subject: [PATCH 45/57] (feat) remove agents --- agents/backpack_mm/AGENT.md | 74 - .../backpack_binance_funding_comparison.py | 257 --- .../routines/backpack_funding_compare.py | 0 .../routines/backpack_mm_volume_analysis.py | 0 .../routines/backpack_volume_scan.py | 257 --- .../skills/backpack_perp_mm_deploy/SKILL.md | 94 - .../config_backpack_aggressive.md | 94 - .../strategies/backpack_perp_mm/strategy.md | 69 - agents/brigado/AGENT.md | 16 - agents/brigado/routines/bot_report.py | 1752 ----------------- agents/brigado/routines/brl_dashboard.py | 573 ------ agents/brigado/routines/btc_brl_arbitrage.py | 608 ------ agents/brigado/routines/config_analyzer.py | 772 -------- agents/brigado/routines/spot_markets.py | 458 ----- .../brigado/routines/toxic_flow_detector.py | 0 agents/brigado/strategies/brl_mm/strategy.md | 32 - 16 files changed, 5056 deletions(-) delete mode 100644 agents/backpack_mm/AGENT.md delete mode 100644 agents/backpack_mm/routines/backpack_binance_funding_comparison.py delete mode 100644 agents/backpack_mm/routines/backpack_funding_compare.py delete mode 100644 agents/backpack_mm/routines/backpack_mm_volume_analysis.py delete mode 100644 agents/backpack_mm/routines/backpack_volume_scan.py delete mode 100644 agents/backpack_mm/skills/backpack_perp_mm_deploy/SKILL.md delete mode 100644 agents/backpack_mm/skills/backpack_perp_mm_deploy/config_backpack_aggressive.md delete mode 100644 agents/backpack_mm/strategies/backpack_perp_mm/strategy.md delete mode 100644 agents/brigado/AGENT.md delete mode 100644 agents/brigado/routines/bot_report.py delete mode 100644 agents/brigado/routines/brl_dashboard.py delete mode 100644 agents/brigado/routines/btc_brl_arbitrage.py delete mode 100644 agents/brigado/routines/config_analyzer.py delete mode 100644 agents/brigado/routines/spot_markets.py delete mode 100644 agents/brigado/routines/toxic_flow_detector.py delete mode 100644 agents/brigado/strategies/brl_mm/strategy.md diff --git a/agents/backpack_mm/AGENT.md b/agents/backpack_mm/AGENT.md deleted file mode 100644 index 79987b40f..000000000 --- a/agents/backpack_mm/AGENT.md +++ /dev/null @@ -1,74 +0,0 @@ ---- -name: Backpack MM -description: Backpack Market Maker Program specialist — volume analysis, market selection, - bot deployment, and ongoing MM operations on Backpack perps -agent_key: claude-acp:sonnet -tools: [] -when_to_consult: When the user wants to analyze Backpack exchange markets, deploy - or manage MM bots on Backpack perps, understand volume requirements for the MM program, - check bot performance against program thresholds, or rotate markets for the Backpack - MM Rewards Program -server_required: true -server_name: local -created_by: 481175164 -created_at: '2026-07-30T06:45:22.685798+00:00' ---- - -# Backpack MM Agent - -You are a specialist in the Backpack Exchange Market Maker Program. You analyze the volume landscape, select markets, deploy and manage pmm_mister bots, and track program qualification metrics. - -## What you do -- Analyze 24h volume per market on Backpack (spot + perpetuals) -- Compute how much maker volume is needed to reach program thresholds -- Assess spread / uptime requirements per market -- Recommend which markets are most tractable given capital constraints -- Deploy and manage pmm_mister bots on Backpack perps -- Monitor bot performance and adjust configs for program compliance -- Rotate markets when volume shifts make qualification harder/easier - -## Program knowledge — Backpack Market Maker Program - -**Reward pool:** $300,000/month — $200k perps, $100k spot - -**Tier score** = max(Total Maker Share, Adjusted Maker Share) -- Total Maker Share = user maker vol / total maker vol (no multipliers) -- Adjusted Maker Share = user adjusted vol / total adjusted vol (lower-liquidity markets get multipliers) - -**Blended Score** (determines reward share per market): -- (Volume Score × 0.8) + (Liquidity Score × 0.2) - -**Volume Score** = user maker vol / total maker vol on that market - -**Liquidity Score** = (Avg Order Size / Avg Spread) × Uptime Score -- Orders must be within **100bps from mid** to count -- Avg order size must be **≥ $2,000** - -**Minimum thresholds to receive rewards on a market:** -- Volume Score ≥ 1.5% -- Avg order size ≥ $2,000 -- At least 1% of total allocation - -**First month:** MM5 maker fees (best tier) regardless of volume. - -## Bot configuration - -Read the `backpack_perp_mm_deploy` skill before any deployment or config change. -The adapted aggressive config uses: -- Connector: `backpack_perpetual` -- **Position mode: ONEWAY** — universal default for all pmm_mister configs on any connector; HEDGE only if explicitly requested. Backpack specifically does not support HEDGE at all (errors on startup) -- 2 levels weighted 2:1, spreads 6bps/15bps -- 300s effectivization, 30s cooldown -- LIMIT_MAKER orders (type 3) for maker credit -- 2x leverage, 5% global SL -- $20K total_amount_quote per controller (min for $2K+ orders) - -## How you answer -- Lead with the key number or recommendation -- Use key: value format, not prose paragraphs -- Always cite the data source and timestamp -- Flag when data is estimated vs fetched live -- If a routine is available, call it and reason over its output — don't guess - -## Routines -- `backpack_volume_scan` — 24h perp volume scan with MM program thresholds diff --git a/agents/backpack_mm/routines/backpack_binance_funding_comparison.py b/agents/backpack_mm/routines/backpack_binance_funding_comparison.py deleted file mode 100644 index b1b9473a9..000000000 --- a/agents/backpack_mm/routines/backpack_binance_funding_comparison.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Compare funding rates between Backpack and Binance perpetuals — interval-normalized carry comparison.""" - -import asyncio -import logging - -import aiohttp -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from condor.reports import ReportBuilder -from routines.base import RoutineResult - -logger = logging.getLogger(__name__) - -CATEGORY = "Arbitrage" - -BACKPACK_API = "https://api.backpack.exchange/api/v1" -BINANCE_FAPI = "https://fapi.binance.com/fapi/v1" - -HOURS_PER_YEAR = 8760 - - -class Config(BaseModel): - """Compare Backpack vs Binance perpetual funding rates, normalized by each exchange's payment interval.""" - - backpack_interval_h: int = Field( - default=1, description="Backpack funding interval in hours (default: 1h)" - ) - binance_interval_h: int = Field( - default=8, description="Binance funding interval in hours (default: 8h)" - ) - min_spread_apr_pct: float = Field( - default=10.0, description="Min annualized spread % to highlight as opportunity" - ) - top_n: int = Field(default=20, description="Max rows in opportunities table") - - -def _normalize_base(base: str) -> str: - """Strip kilo-denomination prefix for cross-exchange matching (1000SHIB / kSHIB → SHIB).""" - if base.startswith("1000"): - return base[4:] - if base and base[0] == "k" and len(base) > 1 and base[1].isupper(): - return base[1:] - return base - - -async def _fetch_backpack_rates( - session: aiohttp.ClientSession, -) -> dict[str, tuple[float, str]]: - """Returns {canonical_base: (rate_per_payment, raw_base)} for all Backpack perp markets.""" - async with session.get(f"{BACKPACK_API}/markets") as resp: - markets = await resp.json() - - perp_symbols = [m["symbol"] for m in markets if m.get("marketType") == "PERP"] - - sem = asyncio.Semaphore(8) - - async def _fetch_one(symbol: str): - async with sem: - try: - async with session.get( - f"{BACKPACK_API}/fundingRates", - params={"symbol": symbol, "limit": 1}, - ) as r: - data = await r.json() - if isinstance(data, list) and data: - rate = data[0].get("fundingRate") - if rate is not None: - return symbol, float(rate) - except Exception as e: - logger.warning(f"Backpack funding fetch failed for {symbol}: {e}") - return symbol, None - - results = await asyncio.gather(*[_fetch_one(s) for s in perp_symbols]) - - out: dict[str, tuple[float, str]] = {} - for symbol, rate in results: - if rate is not None: - raw_base = symbol.replace("_USDC_PERP", "").replace("_USDT_PERP", "") - canonical = _normalize_base(raw_base) - out[canonical] = (rate, raw_base) - - return out - - -async def _fetch_binance_rates(session: aiohttp.ClientSession) -> dict[str, float]: - """Returns {canonical_base: rate_per_payment} for all Binance USDT perp markets.""" - async with session.get(f"{BINANCE_FAPI}/premiumIndex") as resp: - data = await resp.json() - - out: dict[str, float] = {} - for item in data: - symbol = item.get("symbol", "") - if not symbol.endswith("USDT"): - continue - raw_base = symbol[:-4] - canonical = _normalize_base(raw_base) - rate = item.get("lastFundingRate") - if rate is not None: - out[canonical] = float(rate) - - return out - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: - async with aiohttp.ClientSession() as session: - bp_raw, bn_raw = await asyncio.gather( - _fetch_backpack_rates(session), - _fetch_binance_rates(session), - ) - - common = sorted(set(bp_raw) & set(bn_raw)) - - def to_apr(rate_per_payment: float, interval_h: int) -> float: - """Convert a per-payment rate to annualized rate (APR).""" - payments_per_year = HOURS_PER_YEAR / interval_h - return rate_per_payment * payments_per_year - - rows = [] - for base in common: - bp_rate_raw, raw_base = bp_raw[base] - bn_rate_raw = bn_raw[base] - - bp_apr = to_apr(bp_rate_raw, config.backpack_interval_h) - bn_apr = to_apr(bn_rate_raw, config.binance_interval_h) - spread_apr = bp_apr - bn_apr - - # Per-hour rates for a granular comparison column - bp_per_h_bps = (bp_rate_raw / config.backpack_interval_h) * 10000 - bn_per_h_bps = (bn_rate_raw / config.binance_interval_h) * 10000 - spread_per_h_bps = bp_per_h_bps - bn_per_h_bps - - if spread_apr > 0.0001: - trade = "Long BN / Short BP" - elif spread_apr < -0.0001: - trade = "Long BP / Short BN" - else: - trade = "Neutral" - - rows.append( - { - "Asset": base, - "BP Symbol": raw_base, - "BP APR": f"{bp_apr * 100:.2f}%", - "BN APR": f"{bn_apr * 100:.2f}%", - "Spread APR": f"{spread_apr * 100:+.2f}%", - "Spread bps/h": f"{spread_per_h_bps:+.3f}", - "Trade": trade, - "_spread_apr": spread_apr, - "_spread_abs_apr": abs(spread_apr), - "_bp_apr": bp_apr, - "_bn_apr": bn_apr, - "_spread_per_h_bps": spread_per_h_bps, - } - ) - - rows.sort(key=lambda r: r["_spread_abs_apr"], reverse=True) - - min_spread = config.min_spread_apr_pct / 100 - opps = [r for r in rows if r["_spread_abs_apr"] >= min_spread] - top_rows = opps[: config.top_n] - - table_cols = [ - "Asset", - "BP Symbol", - "BP APR", - "BN APR", - "Spread APR", - "Spread bps/h", - "Trade", - ] - clean_top = [{k: r[k] for k in table_cols} for r in top_rows] - clean_all = [{k: r[k] for k in table_cols} for r in rows[:60]] - - # ── Report ───────────────────────────────────────────────────────────── - builder = ReportBuilder("Backpack vs Binance Funding (Interval-Normalized)") - builder.source("routine", "backpack_binance_funding_comparison") - builder.tags(["funding", "arbitrage", "backpack", "binance", "carry"]) - builder.manual_order() - - builder.section("01 / OVERVIEW") - builder.kpi("Backpack Perps", str(len(bp_raw))) - builder.kpi("Binance Perps", str(len(bn_raw))) - builder.kpi("Common Markets", str(len(common))) - builder.kpi(f"Opps ≥{config.min_spread_apr_pct}% APR", str(len(opps))) - builder.kpi("BP Interval", f"{config.backpack_interval_h}h") - builder.kpi("BN Interval", f"{config.binance_interval_h}h") - - builder.markdown( - f"Rates are **annualized**: Backpack raw rate ÷ {config.backpack_interval_h}h × 8,760 h/yr; " - f"Binance raw rate ÷ {config.binance_interval_h}h × 8,760 h/yr. " - f"**Spread bps/h** = (BP per-hour rate − BN per-hour rate) × 10,000 — " - f"the carry you earn or pay per hour of holding the delta-neutral pair." - ) - - dataset_rows = [ - { - "asset": r["Asset"], - "bp_apr_pct": round(r["_bp_apr"] * 100, 2), - "bn_apr_pct": round(r["_bn_apr"] * 100, 2), - "spread_apr_pct": round(r["_spread_apr"] * 100, 2), - "abs_spread_apr_pct": round(r["_spread_abs_apr"] * 100, 2), - "spread_bps_per_h": round(r["_spread_per_h_bps"], 3), - } - for r in rows - ] - builder.dataset("funding", dataset_rows) - builder.chart( - "bar", - "Spread (APR %) by Asset", - "funding", - "asset", - "spread_apr_pct", - y_label="Spread APR (%)", - x_label="Asset", - reference_lines=[{"axis": "y", "value": 0, "label": "Zero"}], - ) - - builder.section( - "02 / OPPORTUNITIES", - f"Sorted by |spread APR|. Threshold: {config.min_spread_apr_pct}% APR. " - f"BP pays every {config.backpack_interval_h}h, BN every {config.binance_interval_h}h.", - ) - if clean_top: - builder.table(clean_top, table_cols) - else: - builder.markdown( - f"No common markets with |spread| ≥ {config.min_spread_apr_pct}% APR." - ) - - builder.section("03 / ALL COMMON MARKETS", "Full list sorted by |spread APR|.") - builder.table(clean_all, table_cols) - - report_id = await builder.save() - - # ── Inline summary ────────────────────────────────────────────────────── - top3 = top_rows[:3] - lines = [ - f"Backpack vs Binance Funding (BP={config.backpack_interval_h}h / BN={config.binance_interval_h}h intervals)", - f"{len(common)} common markets | {len(opps)} opps ≥{config.min_spread_apr_pct}% APR", - "", - ] - if top3: - for r in top3: - lines.append( - f" {r['Asset']:8s} BP={r['BP APR']:>9s} BN={r['BN APR']:>9s}" - f" spread={r['Spread APR']:>9s} → {r['Trade']}" - ) - else: - lines.append(" No opportunities above threshold.") - lines.append(f"\nReport: {report_id}") - - return RoutineResult( - text="\n".join(lines), - table_data=clean_top[:10], - table_columns=table_cols, - ) diff --git a/agents/backpack_mm/routines/backpack_funding_compare.py b/agents/backpack_mm/routines/backpack_funding_compare.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/agents/backpack_mm/routines/backpack_mm_volume_analysis.py b/agents/backpack_mm/routines/backpack_mm_volume_analysis.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/agents/backpack_mm/routines/backpack_volume_scan.py b/agents/backpack_mm/routines/backpack_volume_scan.py deleted file mode 100644 index 5d7373437..000000000 --- a/agents/backpack_mm/routines/backpack_volume_scan.py +++ /dev/null @@ -1,257 +0,0 @@ -"""Scan Backpack Exchange perp markets: 24h volume, MM program thresholds, and tractability ranking.""" - -import logging - -import aiohttp -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from condor.reports import ReportBuilder -from routines.base import RoutineResult - -logger = logging.getLogger(__name__) - -CATEGORY = "Market Data" - -BACKPACK_API = "https://api.backpack.exchange/api/v1" - - -class Config(BaseModel): - """Backpack perp volume scan — MM program threshold analysis""" - - target_share_pct: float = Field( - default=0.5, description="Target maker volume share % (e.g. 0.5 = 0.5%)" - ) - min_volume_score_pct: float = Field( - default=1.5, description="Min Volume Score % to qualify on a market" - ) - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: - async with aiohttp.ClientSession() as session: - # Fetch tickers and markets in parallel - async with session.get(f"{BACKPACK_API}/tickers") as resp: - if resp.status != 200: - return f"Backpack API error: {resp.status}" - tickers = await resp.json() - - async with session.get(f"{BACKPACK_API}/markets") as resp: - if resp.status != 200: - return f"Backpack markets API error: {resp.status}" - markets = await resp.json() - - # Build market info lookup - market_info = {m["symbol"]: m for m in markets if m.get("marketType") == "PERP"} - - # Filter perp tickers and parse volumes - perps = [] - for t in tickers: - sym = t.get("symbol", "") - if not sym.endswith("_PERP"): - continue - quote_vol = float(t.get("quoteVolume", 0)) - last_price = float(t.get("lastPrice", 0)) - trades = int(t.get("trades", 0)) - price_change_pct = float(t.get("priceChangePercent", 0)) * 100 - base = sym.replace("_USDC_PERP", "").replace("k", "") - - # Get margin requirements from market info - info = market_info.get(sym, {}) - imf_base = float(info.get("imfFunction", {}).get("base", 0)) - max_leverage = round(1 / imf_base, 1) if imf_base > 0 else 0 - - perps.append( - { - "symbol": sym, - "base": base, - "quote_volume_24h": quote_vol, - "last_price": last_price, - "trades_24h": trades, - "price_change_pct": round(price_change_pct, 2), - "max_leverage": max_leverage, - } - ) - - if not perps: - return "No perpetual markets found on Backpack" - - # Sort by volume descending - perps.sort(key=lambda x: x["quote_volume_24h"], reverse=True) - - total_volume = sum(p["quote_volume_24h"] for p in perps) - - # Calculate thresholds - target_share = config.target_share_pct / 100 - min_vol_score = config.min_volume_score_pct / 100 - - target_total_volume = total_volume * target_share # 0.5% of total perp volume - monthly_target = target_total_volume * 30 - - # Per-market analysis - table_data = [] - for p in perps: - vol = p["quote_volume_24h"] - vol_share = (vol / total_volume * 100) if total_volume > 0 else 0 - # Volume needed for 1.5% Volume Score on this market - needed_for_qual = vol * min_vol_score / (1 - min_vol_score) - # Volume needed for target share on this market - needed_for_target = vol * target_share / (1 - target_share) - - table_data.append( - { - "Market": p["base"], - "24h Vol": f"${vol:,.0f}", - "Vol Share": f"{vol_share:.1f}%", - "Trades": f"{p['trades_24h']:,}", - "Price": ( - f"${p['last_price']:,.4f}" - if p["last_price"] < 1 - else f"${p['last_price']:,.2f}" - ), - "24h Chg": f"{p['price_change_pct']:+.1f}%", - "Max Lev": f"{p['max_leverage']:.0f}x", - "Need 1.5% VS": f"${needed_for_qual:,.0f}", - "Need 0.5% Total": f"${needed_for_target:,.0f}", - "_vol": vol, - "_needed_qual": needed_for_qual, - "_needed_target": needed_for_target, - } - ) - - # Build report - builder = ReportBuilder("Backpack Perp Volume Scan") - builder.source("routine", "backpack_volume_scan") - builder.tags(["backpack", "volume", "market-making", "perps"]) - builder.manual_order() - - builder.section("01 / PROGRAM OVERVIEW") - builder.kpi("Total Perp Volume (24h)", f"${total_volume:,.0f}") - builder.kpi("Perp Markets", str(len(perps))) - builder.kpi( - f"Target Share ({config.target_share_pct}%)", f"${target_total_volume:,.0f}/day" - ) - builder.kpi("Monthly Target (30d)", f"${monthly_target:,.0f}") - builder.kpi("Min Vol Score Threshold", f"{config.min_volume_score_pct}%") - - builder.section("02 / VOLUME BY MARKET") - builder.markdown( - "**Need 1.5% VS** = maker volume needed on that market to hit the 1.5% Volume Score minimum.\n\n" - f"**Need {config.target_share_pct}% Total** = maker volume needed on that market alone to reach " - f"{config.target_share_pct}% of total perp maker volume." - ) - - # Interactive dataset - dataset_rows = [] - for p in perps: - vol = p["quote_volume_24h"] - vol_share = (vol / total_volume * 100) if total_volume > 0 else 0 - needed_qual = vol * min_vol_score / (1 - min_vol_score) - needed_target = vol * target_share / (1 - target_share) - dataset_rows.append( - { - "market": p["base"], - "volume_24h": round(vol, 2), - "vol_share_pct": round(vol_share, 2), - "trades_24h": p["trades_24h"], - "last_price": p["last_price"], - "change_pct": p["price_change_pct"], - "max_leverage": p["max_leverage"], - "need_1_5_pct_vs": round(needed_qual, 2), - "need_target_share": round(needed_target, 2), - } - ) - - builder.dataset("perps", dataset_rows) - - builder.chart( - "bar", - "24h Volume by Market (USD)", - "perps", - "market", - "volume_24h", - y_label="Volume (USD)", - x_label="Market", - ) - - # Display table (clean columns without internal fields) - display_cols = [ - "Market", - "24h Vol", - "Vol Share", - "Trades", - "Price", - "24h Chg", - "Max Lev", - "Need 1.5% VS", - f"Need {config.target_share_pct}% Total", - ] - clean_table = [{k: row[k] for k in display_cols} for row in table_data] - builder.table(clean_table, display_cols) - - builder.section("03 / TRACTABILITY RANKING") - builder.markdown( - "Markets sorted by **lowest maker volume needed** to hit 1.5% Volume Score — " - "these are the easiest to qualify on." - ) - - # Sort by needed_for_qual ascending (most tractable first) - tractable = sorted(table_data, key=lambda x: x["_needed_qual"]) - tract_table = [] - for i, row in enumerate(tractable[:15], 1): - tract_table.append( - { - "#": i, - "Market": row["Market"], - "24h Vol": row["24h Vol"], - "Need for 1.5% VS": row["Need 1.5% VS"], - "Daily @ Target": row[f"Need {config.target_share_pct}% Total"], - } - ) - builder.table( - tract_table, ["#", "Market", "24h Vol", "Need for 1.5% VS", "Daily @ Target"] - ) - - builder.section("04 / STRATEGY NOTES") - builder.markdown( - f"**Reward pool**: $200k/month for perps\n\n" - f"**Blended Score** = (Volume Score × 0.8) + (Liquidity Score × 0.2)\n\n" - f"**Volume Score** = your maker vol / total maker vol on that market\n\n" - f"**Liquidity Score** = (Avg Order Size / Avg Spread) × Uptime\n\n" - f"**Minimum thresholds per market**: Volume Score ≥ 1.5%, Avg order ≥ $2,000\n\n" - f"**Tier Score** = max(Total Maker Share, Adjusted Maker Share) — " - f"lower-liquidity markets get multipliers in the adjusted calculation, " - f"making them more rewarding per dollar traded.\n\n" - f"**First month**: MM5 maker fees (best tier) regardless of volume." - ) - - report_id = await builder.save() - - # Build inline summary - top5 = perps[:5] - top5_lines = "\n".join( - f" {p['base']}: ${p['quote_volume_24h']:,.0f} (need ${p['quote_volume_24h'] * min_vol_score / (1 - min_vol_score):,.0f} for 1.5% VS)" - for p in top5 - ) - easiest3 = sorted(perps, key=lambda x: x["quote_volume_24h"])[:3] - easiest_lines = "\n".join( - f" {p['base']}: need only ${p['quote_volume_24h'] * min_vol_score / (1 - min_vol_score):,.0f}/day" - for p in easiest3 - ) - - summary = ( - f"Backpack Perp Volume Scan\n" - f"Total 24h perp volume: ${total_volume:,.0f}\n" - f"Target ({config.target_share_pct}% share): ${target_total_volume:,.0f}/day\n\n" - f"Top 5 by volume:\n{top5_lines}\n\n" - f"Easiest to qualify (1.5% VS):\n{easiest_lines}\n\n" - f"Report: {report_id}" - ) - - return RoutineResult( - text=summary, - table_data=[ - {k: row[k] for k in ["Market", "24h Vol", "Vol Share", "Need 1.5% VS"]} - for row in table_data[:10] - ], - table_columns=["Market", "24h Vol", "Vol Share", "Need 1.5% VS"], - ) diff --git a/agents/backpack_mm/skills/backpack_perp_mm_deploy/SKILL.md b/agents/backpack_mm/skills/backpack_perp_mm_deploy/SKILL.md deleted file mode 100644 index 71f6619c8..000000000 --- a/agents/backpack_mm/skills/backpack_perp_mm_deploy/SKILL.md +++ /dev/null @@ -1,94 +0,0 @@ ---- -name: backpack_perp_mm_deploy -description: End-to-end playbook for deploying pmm_mister bots on Backpack perps optimized - for the MM Rewards Program — adapted aggressive config, volume targets, and program - compliance. -when_to_use: When deploying, configuring, or tuning market making bots on Backpack - perpetuals. Also when asked about Backpack MM program compliance, order sizing, - or spread settings for the rewards program. -created: '2026-08-10T21:14:30Z' -source: agent:backpack_mm ---- - -# Backpack Perp MM Deploy Playbook - -Adapted from the market_making_expert's aggressive pmm_mister profile, -tuned specifically for the **Backpack MM Rewards Program** on perpetuals. - -## Program constraints (hard requirements) - -- **Avg order size ≥ $2,000** — orders below this don't count -- **Orders within 100bps from mid** — spreads must be < 0.01 -- **Volume Score ≥ 1.5%** per market to receive rewards -- **Post-only orders** — must be LIMIT_MAKER (type 3) for maker volume -- **Uptime matters** — Liquidity Score = (Avg Order Size / Avg Spread) × Uptime - -## Config template — Backpack MM Aggressive - -Fetch the full config from the companion file: -``` -manage_skill(action="read_file", name="backpack_perp_mm_deploy", file="config_backpack_aggressive.md") -``` - -### Key adaptations vs standard aggressive profile - -| Parameter | Standard Aggressive | Backpack MM | Why | -|-----------|-------------------|-------------|-----| -| total_amount_quote | $500 | **$20,000** | Min $2K orders need larger capital | -| levels | 2 equal | **2 weighted (2:1)** | Front-load L1 for volume, L2 for depth/Liquidity Score | -| spreads | 8bps, 15bps | **6bps, 15bps** | Tighter L1 for fills, wide L2 for depth | -| effectivization_time | 60s | **300s** | More time for TP to fill on thin markets → volume cycling | -| take_profit | 8bps | **6bps** | Fast cycle, viable with MM5 fees (first month) | -| cooldown_time | 30s | **30s** | Prevent adverse stacking on thin markets | -| leverage | 10x | **2x** | Mid-tier perps are volatile, protect capital | -| order_type | 3 | **3** | LIMIT_MAKER mandatory for program credit | - -## Deployment steps - -1. **Run volume scan** — `manage_routines(action="run", name="backpack_volume_scan")` - to get fresh 24h volume data and identify tractable markets. - -2. **Pick markets** — Select 3-5 mid-tier perps where needed maker volume for - 1.5% VS is achievable ($8-15K/day range). Avoid BTC (needs $1M+/day). - -3. **Verify pairs exist on Backpack perps** — Before creating any config, confirm - each ticker actually exists as a perpetual on Backpack by checking - `GET https://api.backpack.exchange/api/v1/markets`. The volume scan may report - spot-style tickers that don't match the actual perp symbol. Known gotchas: - - BONK perp = **kBONK**, SHIB perp = **kSHIB** (kilo-denominated) - - TRX, STABLE — **no perp exists** - - When in doubt, fetch the markets list and grep for the symbol - -4. **Read config template** — fetch `config_backpack_aggressive.md` companion file. - -5. **Adapt per market** — substitute `trading_pair`, verify leverage is within - the market's max (check `imfFunction.base` from /api/v1/markets). - -6. **Set position mode** — Backpack perpetual **only supports ONEWAY** position - mode. Always set `"position_mode": "ONEWAY"`. Never use `HEDGE` — the bot - will error on startup if HEDGE is set. - -7. **Upsert configs** — one config per market via `manage_controllers(action="upsert", target="config", ...)`. - Name convention: `NNN_pmm_backpack_`. - -8. **Deploy single bot** with all controllers: - ``` - manage_bots(action="deploy", bot_name="backpack_mm_perps_", - controllers_config=["config1", "config2", "config3"]) - ``` - -9. **Verify** — `manage_bots(action="status")` to confirm all controllers running. - -## Volume math (per controller) - -- Order size L1: $20K × 0.25 × (2/3) = **$3,333** ✓ (above $2K) -- Order size L2: $20K × 0.25 × (1/3) = **$1,667** (depth, may not count for avg) -- Per complete fill→TP cycle: 2 × $3,333 = **$6,666** maker volume -- Need ~2-3 cycles/day per market for 1.5% VS on $900K market -- At 300s effectivization + 30s cooldown: achievable - -## Monitoring - -- Schedule `backpack_volume_scan` daily to track volume shifts -- Check bot PnL and fill rates via mm_bot_report -- Watch for markets dropping below threshold — may need to rotate diff --git a/agents/backpack_mm/skills/backpack_perp_mm_deploy/config_backpack_aggressive.md b/agents/backpack_mm/skills/backpack_perp_mm_deploy/config_backpack_aggressive.md deleted file mode 100644 index 9f7790b19..000000000 --- a/agents/backpack_mm/skills/backpack_perp_mm_deploy/config_backpack_aggressive.md +++ /dev/null @@ -1,94 +0,0 @@ -# Backpack MM Aggressive Config Template - -Adapted from the MM expert's aggressive profile, tuned for Backpack MM Rewards -Program compliance. All orders are LIMIT_MAKER (post-only) for maker credit. - -## Config (substitute connector/pair/amount per market) - -```json -{ - "controller_type": "generic", - "controller_name": "pmm_mister", - "connector_name": "backpack_perpetual", - "trading_pair": "-USDC", - "total_amount_quote": 20000, - "portfolio_allocation": 0.25, - "target_base_pct": 0.5, - "min_base_pct": 0.3, - "max_base_pct": 0.7, - "buy_spreads": "0.0003,0.0015", - "sell_spreads": "0.0003,0.0015", - "buy_amounts_pct": "2,1", - "sell_amounts_pct": "2,1", - "executor_refresh_time": 20, - "buy_cooldown_time": 30, - "sell_cooldown_time": 30, - "buy_position_effectivization_time": 300, - "sell_position_effectivization_time": 300, - "price_distance_tolerance": 0.0005, - "refresh_tolerance": 0.0003, - "tolerance_scaling": 1.1, - "open_order_type": 3, - "take_profit": 0.0006, - "take_profit_order_type": 3, - "leverage": 2, - "position_mode": "ONEWAY", - "position_side": "BUY", - "max_active_executors_by_level": 2, - "tick_mode": false, - "min_skew": 1.0, - "global_tp_enabled": false, - "global_sl_enabled": true, - "global_stop_loss": 0.05, - "global_sl_activation_from": "target_base", - "global_pnl_reference": "position" -} -``` - -## Position mode — universal default - -**Always use `"position_mode": "ONEWAY"`** for pmm_mister configs on any connector. -This is the universal default — not just a Backpack restriction. HEDGE should only -be used when the user explicitly requests it. - -On Backpack specifically, HEDGE is not supported at all and will error on startup. -On other connectors (Hyperliquid, Binance, etc.), ONEWAY is still the default unless -the user says otherwise. - -## Parameter rationale - -**Order sizing** (meets $2K minimum): -- Level 1: $20K × 0.25 × (2/3) = $3,333 per order -- Level 2: $20K × 0.25 × (1/3) = $1,667 per order (depth) -- Both sides quoting = continuous presence - -**Spreads** (within 100bps program limit): -- L1: 3bps — very tight, maximizes fill rate and volume generation -- L2: 15bps — passive depth, helps Liquidity Score - -**Effectivization** (300s): -- After a fill, the LIMIT_MAKER TP order stays on book for 5 minutes -- On mid-tier perps, price needs time to cycle back for TP fill -- Complete fill→TP cycle = 2× notional in maker volume -- 60s was too short (most positions expired to hold); 300s catches ~50%+ of TPs - -**Take profit** (6bps): -- With MM5 fees (~0.02% maker), round-trip ~0.04% -- 6bps TP gives 1.5× fee coverage — tight but viable -- If Backpack fees are higher than expected, bump to 8bps - -**Leverage** (2x): -- Mid-tier perps (ZEC, PUMP, WLD) can gap 5-10% -- At 2x: 5% move = 10% drawdown → survivable with 5% global SL -- At 5x: 5% move = 25% drawdown → dangerous - -**Cooldown** (30s): -- Prevents stacking fills at similar prices on thin markets -- 15s was too aggressive for $500-900K/day volume markets - -## Adjustments per market - -- **PAXG**: Gold-backed, low vol → can keep 3bps, reduce cooldown to 20s -- **ZEC/WLD**: Moderate vol → use template as-is -- **PUMP**: Higher vol, can gap → monitor closely with 3bps, widen to 5bps if adverse fills spike -- **Leverage**: Always verify max leverage per market (1/imfFunction.base from API) diff --git a/agents/backpack_mm/strategies/backpack_perp_mm/strategy.md b/agents/backpack_mm/strategies/backpack_perp_mm/strategy.md deleted file mode 100644 index 35c9c1b0c..000000000 --- a/agents/backpack_mm/strategies/backpack_perp_mm/strategy.md +++ /dev/null @@ -1,69 +0,0 @@ ---- -name: Backpack Perp MM -description: '' -agent_key: null -skills: [] -default_config: - frequency_sec: 3600 - total_amount_quote: 60000 -default_trading_context: '' -created_by: 481175164 -created_at: '2026-08-10T21:16:55.078013+00:00' ---- - -# Backpack Perp MM Strategy - -Market making on Backpack perpetuals targeting the MM Rewards Program. -Agent pinned to `local` server. - -## Loop playbook - -Each tick: - -1. **Volume check** — Run `backpack_volume_scan` routine to get fresh 24h volumes. - Compare current market volumes against previous tick. Flag markets where volume - dropped below qualification threshold. - -2. **Bot health** — Check `manage_bots(action="status")` for the running bot. - Verify all controllers are in `running` state. Check for errors in logs. - -3. **Performance assessment** — For each controller: - - Current PnL (realized + unrealized) - - Volume generated vs target (need 1.5% VS per market) - - Inventory balance (base_pct within min/max band) - - Fill rate and TP hit rate - -4. **Compliance check** — Verify program metrics: - - Are avg order sizes ≥ $2,000? - - Are spreads within 100bps? - - Is uptime acceptable for Liquidity Score? - -5. **Adjust if needed**: - - If a market's volume dropped and qualification is at risk → consider rotating - to a more tractable market - - If inventory is drifting → adjust target/min/max_base_pct live - - If PnL is bleeding → widen spreads or increase cooldown - - If volume target is not being met → tighten spreads cautiously - -6. **Journal** — Write a summary of the tick: volumes, PnL, compliance status, - any adjustments made. - -## Active deployment - -- Bot: `backpack_mm_perps_20260811-20260810-211539` -- Controllers (all 3bps L1 / 15bps L2, 6bps TP, 2x leverage, $20K each): - - `001_pmm_backpack_ZEC-USDC` — ZEC perp - - `002_pmm_backpack_WLD-USDC` — WLD perp - - `003_pmm_backpack_PUMP-USDC` — PUMP perp - -## Risk limits - -- Max $20K per controller ($60K total across 3 markets) -- 2x leverage -- 5% global stop loss per controller -- No global TP — let volume cycle - -## Skills to read - -- `backpack_perp_mm_deploy` — full config template and deployment steps -- `backpack_volume_scan` routine — fresh volume data diff --git a/agents/brigado/AGENT.md b/agents/brigado/AGENT.md deleted file mode 100644 index b81b6a9d5..000000000 --- a/agents/brigado/AGENT.md +++ /dev/null @@ -1,16 +0,0 @@ ---- -name: Brigado -description: BRL market making research agent. Tracks USDT-BRL spreads, BTC/fiat - volume dynamics, and bot performance for the Brazilian market making operation. -agent_key: claude-acp:sonnet -tools: [] -when_to_consult: '' -server_required: true -created_by: 481175164 -created_at: '2026-05-04T00:00:00+00:00' ---- - -## Brigado - -### Objective -Research and monitoring agent for the Brazilian market making operation. Provides dashboards, volume analysis, and bot performance tracking focused on BRL pairs and fiat market dynamics. diff --git a/agents/brigado/routines/bot_report.py b/agents/brigado/routines/bot_report.py deleted file mode 100644 index 561257bea..000000000 --- a/agents/brigado/routines/bot_report.py +++ /dev/null @@ -1,1752 +0,0 @@ -"""Bot Performance Report — Clean 2-message report with USD values and rebate tracking.""" - -CATEGORY = "Bot Analysis" - -import asyncio -import logging -import os -import re -from collections import defaultdict -from datetime import datetime, timezone - -import plotly.graph_objects as go -from plotly.subplots import make_subplots -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from config_manager import get_client, get_config_manager - -logger = logging.getLogger(__name__) - -REBATE_RATE = 0.00015 # 0.015% of volume — only applies to BRL-quoted pairs - -# Config analysis constants -_CONFIG_SKIP_PARAMS = { - "id", - "controller_type", - "controller_name", - "manual_kill_switch", - "_config_name", -} -_CONFIG_KEY_PARAMS = { - "buy_spreads", - "sell_spreads", - "buy_amounts_pct", - "sell_amounts_pct", - "total_amount_quote", - "stop_loss", - "take_profit", - "time_limit", - "executor_refresh_time", - "cooldown_time", - "leverage", - "dca_spreads", - "dca_amounts", - "trailing_stop", - "executor_activation_bounds", - "top_executor_refresh_time", - "position_rebalance_threshold_pct", -} - - -def _pair_has_rebate(pair: str, quote_currency: str) -> bool: - """Rebate only applies to pairs quoted in the rebate currency (BRL).""" - if not pair or "-" not in pair: - return False - return pair.split("-")[-1].upper() == quote_currency.upper() - - -class Config(BaseModel): - """Generate a formatted performance report for all active bots.""" - - quote_currency: str = Field( - default="BRL", - description="Quote currency of the bots (for USD conversion)", - ) - target_chat_id: int | None = Field( - default=None, - description="Send report to this chat ID instead of the current chat (e.g. a group chat)", - ) - server_name: str = Field( - default="brigado_2", - description="Server to connect to for bot data (bots run on this server)", - ) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def _fmt(n: float, decimals: int = 2) -> str: - if abs(n) >= 1_000: - return f"{n:,.{decimals}f}" - return f"{n:.{decimals}f}" - - -def _sign(n: float) -> str: - return "+" if n >= 0 else "-" - - -def _emoji(n: float) -> str: - return "🟢" if n > 0 else "🔴" if n < 0 else "⚪" - - -def _md_escape(text: str) -> str: - """Escape special characters for Telegram MarkdownV2.""" - special = r"_*[]()~`>#+-=|{}.!" - result = [] - for ch in text: - if ch in special: - result.append(f"\\{ch}") - else: - result.append(ch) - return "".join(result) - - -# --------------------------------------------------------------------------- -# Data fetching -# --------------------------------------------------------------------------- - - -async def _get_bots_data(client) -> dict: - data = await client.bot_orchestration.get_active_bots_status() - if isinstance(data, dict) and "data" in data: - return data["data"] - return {} - - -async def _get_bot_runs(client) -> dict: - runs = {} - try: - data = await client.bot_orchestration.get_bot_runs() - if isinstance(data, dict) and "data" in data: - for run in data["data"]: - if run.get("deployment_status") == "DEPLOYED" and run.get( - "deployed_at" - ): - runs[run["bot_name"]] = run["deployed_at"] - except Exception: - pass - return runs - - -async def _get_prices(client, connector: str, pairs: list[str]) -> dict[str, float]: - try: - result = await client.market_data.get_prices( - connector_name=connector, trading_pairs=",".join(pairs) - ) - return result.get("prices", {}) if result else {} - except Exception as e: - logger.warning(f"Failed to get prices: {e}") - return {} - - -async def _get_balances(client) -> dict: - """Fetch portfolio balances from all accounts.""" - try: - return await client.portfolio.get_state(refresh=False) - except Exception as e: - logger.warning(f"Failed to get balances: {e}") - return {} - - -async def _get_market_volume_since( - client, connector: str, pair: str, since_dt: datetime | None -) -> tuple[float, float]: - """Return (base_volume, quote_volume) accumulated from candles since *since_dt*. - - Uses 1h candles spanning from *since_dt* to now so the market volume window - matches the bot's runtime, giving an accurate market-share calculation. - Falls back to a single 1d candle when *since_dt* is unavailable. - """ - try: - if since_dt: - now = datetime.now(timezone.utc) - days = max(1, int((now - since_dt).total_seconds() / 86400) + 1) - candles = await client.market_data.get_candles_last_days( - connector_name=connector, - trading_pair=pair, - days=days, - interval="1h", - ) - else: - candles = await client.market_data.get_candles( - connector_name=connector, - trading_pair=pair, - interval="1d", - max_records=1, - ) - - if candles: - data = candles if isinstance(candles, list) else candles.get("data", []) - if not data: - return 0.0, 0.0 - - since_ts = since_dt.timestamp() if since_dt else 0 - total_base = 0.0 - total_quote = 0.0 - for row in data: - # Only count candles that start at or after the deploy time - ts = float(row.get("timestamp", 0) or 0) - # Normalise ms → s if needed - if ts > 1e12: - ts /= 1000 - if ts >= since_ts: - total_base += float(row.get("volume", 0) or 0) - total_quote += float(row.get("quote_asset_volume", 0) or 0) - return total_base, total_quote - except Exception as e: - logger.warning(f"Failed to get volume for {pair}: {e}") - return 0.0, 0.0 - - -# --------------------------------------------------------------------------- -# Chart builders (Plotly) -# --------------------------------------------------------------------------- - -_DARK_BG = "#1a1a2e" -_DARK_PAPER = "#16213e" -_GREEN = "#00d4aa" -_RED = "#ff6b6b" -_BLUE = "#58a6ff" -_YELLOW = "#ffdd57" -_COLORS = [ - "#58a6ff", - "#00d4aa", - "#ff6b6b", - "#ffdd57", - "#c084fc", - "#fb923c", - "#38bdf8", - "#f472b6", -] - - -def _chart_layout(title: str, **kwargs) -> dict: - base = dict( - title=dict(text=title, font=dict(color="#e6edf3", size=14), x=0.5), - paper_bgcolor=_DARK_PAPER, - plot_bgcolor=_DARK_BG, - font=dict(color="#ccc", size=11), - margin=dict(l=60, r=80, t=50, b=80), - width=1100, - height=450, - ) - base.update(kwargs) - return base - - -def _quote_is_usd(pair: str) -> bool: - """Check if a pair's quote currency is a USD stablecoin.""" - if not pair or "-" not in pair: - return False - return pair.split("-")[-1].upper() in ("USD", "USDT", "USDC", "BUSD", "FDUSD") - - -def _vol_to_usd(ctrl: dict, to_usd: float) -> float: - """Convert controller volume to USD based on the pair's quote currency.""" - if _quote_is_usd(ctrl.get("pair", "")): - return ctrl["vol"] - return ctrl["vol"] * to_usd - - -def _amount_quote_to_usd(ctrl: dict, to_usd: float) -> float: - """Convert total_amount_quote to USD based on the pair's quote currency.""" - amt = ctrl.get("total_amount_quote", 0) - if _quote_is_usd(ctrl.get("pair", "")): - return amt - return amt * to_usd - - -def _market_vol_to_usd( - pair: str, - vol_base: float, - vol_quote: float, - to_usd: float, - prices: dict[str, float], -) -> float: - """Convert market candle volume to USD for any pair.""" - base = pair.split("-")[0].upper() if "-" in pair else "" - quote = pair.split("-")[-1].upper() if "-" in pair else "" - - # If base is a stablecoin, vol_base is already ~USD - if base in ("USDT", "USDC", "BUSD", "DAI", "FDUSD"): - return vol_base - - # If quote is USD stablecoin, vol_quote is already USD - if quote in ("USD", "USDT", "USDC", "BUSD", "FDUSD"): - return ( - vol_quote - if vol_quote > 0 - else vol_base * (prices.get(f"{base}-USDT", 0) or 0) - ) - - # Non-USD quote (e.g. BRL) — convert quote volume, or use base price - if vol_quote > 0: - return vol_quote * to_usd - bp = prices.get(f"{base}-USDT", 0) or prices.get(f"{base}-USD", 0) - return vol_base * bp if bp else 0 - - -def _short_chart_name(ctrl_id: str, pair: str = "") -> str: - """Clean controller name for chart labels.""" - name = ctrl_id - # Strip common prefixes - name = re.sub(r"^mm_", "", name) - name = re.sub(r"pmm_?mister_?", "", name) - # Strip connector prefixes (binance_, binance_perpetual_, etc.) - name = re.sub(r"^(binance|okx|bybit|kucoin)(_perpetual)?_", "", name) - # Strip pair fragment - if pair: - pair_frag = pair.replace("-", "_") - name = name.replace(pair_frag, "").replace(pair, "") - while "__" in name: - name = name.replace("__", "_") - name = name.strip("_") - name = re.sub(r"config[_]?(\d+)", r"c\1", name) - return name or ctrl_id[:20] - - -def _build_pnl_chart( - controllers: list[dict], - to_usd: float, - quote_currency: str = "BRL", - ctrl_configs: dict | None = None, -) -> go.Figure: - """Horizontal bar chart: PnL breakdown, Total PnL, and Vol/Capital by controller.""" - # Sort by net PnL - sorted_ctrls = sorted(controllers, key=lambda c: c["net"] * to_usd) - names = [_short_chart_name(c["id"], c["pair"]) for c in sorted_ctrls] - realized = [c["realized"] * to_usd for c in sorted_ctrls] - unrealized = [c["unrealized"] * to_usd for c in sorted_ctrls] - rebates = [ - ( - c["vol"] * to_usd * REBATE_RATE - if _pair_has_rebate(c["pair"], quote_currency) - else 0 - ) - for c in sorted_ctrls - ] - totals = [r + u + rb for r, u, rb in zip(realized, unrealized, rebates)] - volumes = [_vol_to_usd(c, to_usd) for c in sorted_ctrls] - - # Capital from configs (converted to USD) - has_capital = False - capitals = [] - if ctrl_configs: - for c in sorted_ctrls: - cfg = ctrl_configs.get(c["id"], {}) - amt = float(cfg.get("total_amount_quote", 0) or 0) - ctrl_for_conv = {"total_amount_quote": amt, "pair": c["pair"]} - capitals.append(_amount_quote_to_usd(ctrl_for_conv, to_usd)) - if amt > 0: - has_capital = True - - # Vol/Capital turnover ratio - vol_cap_ratios = [] - if has_capital: - for v, cap in zip(volumes, capitals): - vol_cap_ratios.append(v / cap if cap > 0 else 0) - - ncols = 3 if has_capital else 2 - col_widths = [0.40, 0.30, 0.30] if has_capital else [0.55, 0.45] - subtitles = ( - ["PnL Breakdown (USD)", "Total PnL (USD)", "Vol / Capital"] - if has_capital - else ["PnL Breakdown (USD)", "Total PnL (USD)"] - ) - - fig = make_subplots( - rows=1, - cols=ncols, - shared_yaxes=True, - column_widths=col_widths, - horizontal_spacing=0.03, - subplot_titles=subtitles, - ) - - # Col 1: PnL breakdown (stacked) - fig.add_trace( - go.Bar( - y=names, - x=realized, - orientation="h", - name="Realized", - marker_color=_GREEN, - opacity=0.85, - ), - row=1, - col=1, - ) - fig.add_trace( - go.Bar( - y=names, - x=unrealized, - orientation="h", - name="Unrealized", - marker_color=_BLUE, - opacity=0.85, - ), - row=1, - col=1, - ) - fig.add_trace( - go.Bar( - y=names, - x=rebates, - orientation="h", - name="Rebate", - marker_color=_YELLOW, - opacity=0.7, - ), - row=1, - col=1, - ) - - # Col 2: Total PnL (single bar per controller) - total_colors = [_GREEN if t >= 0 else _RED for t in totals] - fig.add_trace( - go.Bar( - y=names, - x=totals, - orientation="h", - name="Total", - marker_color=total_colors, - opacity=0.9, - text=[f"{'+'if t>=0 else ''}{t:.1f}" for t in totals], - textposition="inside", - textfont=dict(size=10, color="white"), - insidetextanchor="middle", - showlegend=False, - ), - row=1, - col=2, - ) - - # Col 3: Vol/Capital turnover - if has_capital: - fig.add_trace( - go.Bar( - y=names, - x=vol_cap_ratios, - orientation="h", - name="Vol/Capital", - marker_color="#8b5cf6", - opacity=0.85, - text=[f"{r:.0f}x" if r > 0 else "-" for r in vol_cap_ratios], - textposition="outside", - textfont=dict(size=10), - showlegend=False, - ), - row=1, - col=3, - ) - - height = max(350, len(names) * 40 + 100) - fig.update_layout( - **_chart_layout("PnL Breakdown · Total · Vol/Capital", height=height), - barmode="relative", - legend=dict(orientation="h", y=-0.12, x=0.5, xanchor="center"), - ) - fig.update_xaxes(gridcolor="#2a2a4a", title="USD", row=1, col=1) - fig.update_xaxes(gridcolor="#2a2a4a", title="USD", row=1, col=2) - if has_capital: - fig.update_xaxes(gridcolor="#2a2a4a", title="Turnover (x)", row=1, col=3) - fig.update_yaxes(gridcolor="#2a2a4a", row=1, col=1) - # Fix subplot title colors - for ann in fig.layout.annotations: - ann.font = dict(color="#aaa", size=11) - return fig - - -def _build_pair_overview_chart( - controllers: list[dict], - to_usd: float, - volumes_since: dict[str, tuple[float, float]], - prices: dict[str, float], - quote_currency: str = "BRL", -) -> go.Figure: - """Unified horizontal bar chart: PnL, Volume, and Market Share by pair.""" - # Aggregate data per pair - pair_data: dict[str, dict] = defaultdict( - lambda: {"realized": 0, "unrealized": 0, "vol": 0} - ) - for c in controllers: - key = c["pair"] or "unknown" - pair_data[key]["realized"] += c["realized"] * to_usd - pair_data[key]["unrealized"] += c["unrealized"] * to_usd - pair_data[key]["vol"] += _vol_to_usd(c, to_usd) - - # Sort pairs by total volume (highest at top) - pairs = sorted(pair_data.keys(), key=lambda p: pair_data[p]["vol"]) - - # Compute market share data - pair_market: dict[str, dict] = {} - for pair in pairs: - vol_data = volumes_since.get(pair, (0, 0)) - vol_base, vol_quote = vol_data if isinstance(vol_data, tuple) else (vol_data, 0) - mkt_usd = ( - _market_vol_to_usd(pair, vol_base, vol_quote, to_usd, prices) - if (vol_base > 0 or vol_quote > 0) - else 0 - ) - our_vol = pair_data[pair]["vol"] - share = (our_vol / mkt_usd * 100) if mkt_usd > 0 else 0 - pair_market[pair] = {"market": mkt_usd, "ours": our_vol, "share": share} - - has_market = any(d["market"] > 0 for d in pair_market.values()) - ncols = 3 if has_market else 2 - col_widths = [0.4, 0.35, 0.25] if has_market else [0.5, 0.5] - subtitles = ( - ["PnL (USD)", "Volume (USD)", "Market Share"] - if has_market - else ["PnL (USD)", "Volume (USD)"] - ) - - fig = make_subplots( - rows=1, - cols=ncols, - shared_yaxes=True, - column_widths=col_widths, - horizontal_spacing=0.04, - subplot_titles=subtitles, - ) - - # Col 1: PnL (realized + unrealized + rebate) - fig.add_trace( - go.Bar( - y=pairs, - x=[pair_data[p]["realized"] for p in pairs], - orientation="h", - name="Realized", - marker_color=_GREEN, - opacity=0.85, - ), - row=1, - col=1, - ) - fig.add_trace( - go.Bar( - y=pairs, - x=[pair_data[p]["unrealized"] for p in pairs], - orientation="h", - name="Unrealized", - marker_color=_BLUE, - opacity=0.85, - ), - row=1, - col=1, - ) - fig.add_trace( - go.Bar( - y=pairs, - x=[ - ( - pair_data[p]["vol"] * REBATE_RATE - if _pair_has_rebate(p, quote_currency) - else 0 - ) - for p in pairs - ], - orientation="h", - name="Rebate", - marker_color=_YELLOW, - opacity=0.7, - ), - row=1, - col=1, - ) - - # Col 2: Volume - fig.add_trace( - go.Bar( - y=pairs, - x=[pair_data[p]["vol"] for p in pairs], - orientation="h", - name="Volume", - marker_color="#8b5cf6", - opacity=0.8, - ), - row=1, - col=2, - ) - - # Col 3: Market Share as percentage bar - if has_market: - shares = [pair_market[p]["share"] for p in pairs] - fig.add_trace( - go.Bar( - y=pairs, - x=shares, - orientation="h", - name="Market Share", - marker_color=[_BLUE if s > 0 else "#30363d" for s in shares], - opacity=0.85, - text=[f"{s:.2f}%" if s > 0 else "N/A" for s in shares], - textposition="outside", - textfont=dict(size=11, color=_BLUE), - hovertemplate="%{y}
Share: %{x:.2f}%
" - "Ours: $%{customdata[0]:,.0f}
" - "Market: $%{customdata[1]:,.0f}", - customdata=[ - [pair_market[p]["ours"], pair_market[p]["market"]] for p in pairs - ], - ), - row=1, - col=ncols, - ) - - height = max(350, len(pairs) * 55 + 120) - fig.update_layout( - **_chart_layout("Pair Overview: PnL · Volume · Market Share", height=height), - barmode="relative", - legend=dict(orientation="h", y=-0.12, x=0.5, xanchor="center"), - ) - for col in range(1, ncols + 1): - fig.update_xaxes(gridcolor="#2a2a4a", row=1, col=col) - if has_market: - fig.update_xaxes(title="%", row=1, col=ncols) - fig.update_yaxes(gridcolor="#2a2a4a", row=1, col=1) - # Fix subplot title colors - for ann in fig.layout.annotations: - ann.font = dict(color="#aaa", size=11) - return fig - - -# --------------------------------------------------------------------------- -# Config analysis helpers -# --------------------------------------------------------------------------- - - -def _cfg_fmt_value(v) -> str: - if v is None: - return "—" - if isinstance(v, bool): - return "✓" if v else "✗" - if isinstance(v, float): - if v < 0.001: - return f"{v:.5f}" - if v < 0.01: - return f"{v:.4f}" - return f"{v:.3f}" if v < 10 else f"{v:.1f}" - if isinstance(v, list): - return ", ".join(_cfg_fmt_value(x) for x in v) - if isinstance(v, dict): - return " | ".join(f"{k}={_cfg_fmt_value(vv)}" for k, vv in v.items()) - return str(v) - - -def _cfg_normalize(v): - if v is None: - return None - if isinstance(v, str): - try: - return float(v) - except ValueError: - return v.strip().lower() - if isinstance(v, list): - return [_cfg_normalize(x) for x in v] - if isinstance(v, dict): - return {k: _cfg_normalize(vv) for k, vv in v.items()} - return v - - -def _build_config_parallel_coords( - all_configs: dict, running_ids: set -) -> go.Figure | None: - NUMERIC_AXES = [ - ("total_amount_quote", "Capital", None), - ( - "buy_spreads", - "Buy Spread L1 %", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "sell_spreads", - "Sell Spread L1 %", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "buy_spreads", - "Buy Spread L2 %", - lambda v: ( - (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None - ), - ), - ( - "sell_spreads", - "Sell Spread L2 %", - lambda v: ( - (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None - ), - ), - ("executor_refresh_time", "Refresh Time", None), - ("cooldown_time", "Cooldown", None), - ("leverage", "Leverage", None), - ("stop_loss", "Stop Loss", None), - ("take_profit", "Take Profit", None), - ("time_limit", "Time Limit", None), - ("top_executor_refresh_time", "Top Refresh", None), - ( - "dca_spreads", - "DCA Spread L1", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "dca_amounts", - "DCA Amt L1", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "trailing_stop", - "Trail Stop", - lambda v: ( - (v.get("activation_price_delta") if isinstance(v, dict) else v) - if v - else None - ), - ), - ( - "executor_activation_bounds", - "Activation Bounds", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ] - - rows = [] - labels = [] - for label, cfg in all_configs.items(): - clean_label = label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", "") - is_running = clean_label in running_ids or label.startswith("\U0001f7e2") - pair = cfg.get("trading_pair", "unknown") - row = {"_running": 1 if is_running else 0, "_pair": pair} - for param, axis_name, extractor in NUMERIC_AXES: - raw = cfg.get(param) - val = extractor(raw) if extractor else raw - try: - row[axis_name] = float(val) if val is not None else None - except (TypeError, ValueError): - row[axis_name] = None - rows.append(row) - labels.append(_short_chart_name(clean_label, pair)) - - if len(rows) < 2: - return None - - valid_axes = [] - for _, axis_name, _ in NUMERIC_AXES: - vals = [r[axis_name] for r in rows if r[axis_name] is not None] - if len(vals) >= 2 and len(set(vals)) > 1: - valid_axes.append(axis_name) - - if len(valid_axes) < 2: - return None - - unique_pairs = sorted(set(r["_pair"] for r in rows)) - pair_to_idx = {p: i for i, p in enumerate(unique_pairs)} - color_values = [pair_to_idx[r["_pair"]] for r in rows] - - n_pairs = len(unique_pairs) - if n_pairs == 1: - colorscale = [[0, _BLUE], [1, _BLUE]] - else: - colorscale = [ - [i / (n_pairs - 1), _COLORS[i % len(_COLORS)]] for i in range(n_pairs) - ] - - for axis_name in valid_axes: - vals = [r[axis_name] for r in rows if r[axis_name] is not None] - fill = min(vals) if vals else 0 - for r in rows: - if r[axis_name] is None: - r[axis_name] = fill - - dimensions = [] - for axis_name in valid_axes: - vals = [r[axis_name] for r in rows] - if "%" in axis_name: - vals = [v * 100 for v in vals] - dimensions.append( - dict( - label=axis_name, - values=vals, - range=( - [min(vals), max(vals)] - if min(vals) != max(vals) - else [min(vals) - 1, max(vals) + 1] - ), - ) - ) - - fig = go.Figure( - data=go.Parcoords( - line=dict(color=color_values, colorscale=colorscale, showscale=False), - dimensions=dimensions, - customdata=labels, - ) - ) - for i, pair in enumerate(unique_pairs): - fig.add_trace( - go.Scatter( - x=[None], - y=[None], - mode="markers", - marker=dict(size=10, color=_COLORS[i % len(_COLORS)]), - name=pair, - showlegend=True, - ) - ) - fig.update_layout( - **_chart_layout( - f"Config Parameter Space ({len(rows)} configs, {len(valid_axes)} axes)", - width=1400, - height=600, - ), - legend=dict( - orientation="h", y=-0.05, x=0.5, xanchor="center", font=dict(size=12) - ), - ) - return fig - - -def _build_config_spread_chart(all_configs: dict) -> go.Figure | None: - def _to_list(v) -> list: - if v is None: - return [] - if isinstance(v, (int, float)): - return [float(v)] - if isinstance(v, list): - return [float(x) for x in v] - return [] - - items = [] - for label, cfg in all_configs.items(): - buy = _to_list(cfg.get("buy_spreads")) - sell = _to_list(cfg.get("sell_spreads")) - if buy or sell: - short = _short_chart_name( - label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", ""), - cfg.get("trading_pair", ""), - ) - items.append( - { - "name": short, - "pair": cfg.get("trading_pair", ""), - "buy_l1": buy[0] * 100 if len(buy) >= 1 else 0, - "sell_l1": sell[0] * 100 if len(sell) >= 1 else 0, - "buy_l2": buy[1] * 100 if len(buy) >= 2 else 0, - "sell_l2": sell[1] * 100 if len(sell) >= 2 else 0, - } - ) - - if not items: - return None - - items.sort(key=lambda c: (c["pair"], c["name"])) - names = [c["name"] for c in items] - - fig = go.Figure() - fig.add_trace( - go.Bar( - x=names, - y=[c["buy_l1"] for c in items], - name="Buy L1", - marker_color=_GREEN, - opacity=0.9, - ) - ) - fig.add_trace( - go.Bar( - x=names, - y=[c["sell_l1"] for c in items], - name="Sell L1", - marker_color=_RED, - opacity=0.9, - ) - ) - if any(c["buy_l2"] > 0 or c["sell_l2"] > 0 for c in items): - fig.add_trace( - go.Bar( - x=names, - y=[c["buy_l2"] for c in items], - name="Buy L2", - marker_color=_GREEN, - opacity=0.5, - ) - ) - fig.add_trace( - go.Bar( - x=names, - y=[c["sell_l2"] for c in items], - name="Sell L2", - marker_color=_RED, - opacity=0.5, - ) - ) - - fig.update_layout( - **_chart_layout("Spread Levels by Controller (%)", width=1200, height=500), - barmode="group", - yaxis=dict(gridcolor="#2a2a4a", title="Spread %"), - xaxis=dict(gridcolor="#2a2a4a", tickangle=-45), - legend=dict(orientation="h", y=1.1, x=0.5, xanchor="center"), - ) - fig.update_layout(margin=dict(l=60, r=40, t=60, b=120)) - return fig - - -def _build_config_capital_chart(all_configs: dict) -> go.Figure | None: - data = [] - for label, cfg in all_configs.items(): - amt = float(cfg.get("total_amount_quote", 0) or 0) - if amt <= 0: - continue - short = _short_chart_name( - label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", ""), - cfg.get("trading_pair", ""), - ) - data.append({"name": short, "pair": cfg.get("trading_pair", ""), "amount": amt}) - - if not data: - return None - - data.sort(key=lambda d: (d["pair"], d["amount"])) - unique_pairs = list(dict.fromkeys(d["pair"] for d in data)) - pair_colors = {p: _COLORS[i % len(_COLORS)] for i, p in enumerate(unique_pairs)} - - fig = go.Figure() - for pair in unique_pairs: - pair_data = [d for d in data if d["pair"] == pair] - fig.add_trace( - go.Bar( - y=[d["name"] for d in pair_data], - x=[d["amount"] for d in pair_data], - orientation="h", - name=pair, - marker_color=pair_colors[pair], - opacity=0.85, - text=[f"{d['amount']:,.0f}" for d in pair_data], - textposition="auto", - ) - ) - - height = max(400, len(data) * 22 + 100) - fig.update_layout( - **_chart_layout( - "Capital Allocation by Controller (quote)", width=1200, height=height - ), - barmode="stack", - xaxis=dict(gridcolor="#2a2a4a", title="Amount (quote)"), - yaxis=dict(gridcolor="#2a2a4a"), - legend=dict(orientation="h", y=1.05, x=0.5, xanchor="center"), - ) - fig.update_layout(margin=dict(l=150, r=40, t=60, b=50)) - return fig - - -# --------------------------------------------------------------------------- -# Report builder -# --------------------------------------------------------------------------- - - -def _build_report( - bots_data: dict, - bot_runs: dict, - prices: dict[str, float], - volumes_since: dict[str, tuple[float, float]], - quote_usd_rate: float, - quote_currency: str = "BRL", - ctrl_configs: dict | None = None, - balances: dict | None = None, -) -> list[str]: - ctrl_configs = ctrl_configs or {} - - # ---- Parse controller data ---- - controllers = [] - - for bot_name, bot_data in bots_data.items(): - if isinstance(bot_data, str): - continue - - performance = {} - if isinstance(bot_data, dict): - performance = bot_data.get("performance", {}) - elif isinstance(bot_data, list): - performance = { - f"ctrl_{i}": {"performance": c} for i, c in enumerate(bot_data) - } - - for ctrl_name, ctrl_data in performance.items(): - if not isinstance(ctrl_data, dict): - continue - perf = ctrl_data.get("performance", ctrl_data) - cfg = ctrl_configs.get(ctrl_name, {}) - trading_pair = cfg.get("trading_pair", "") or perf.get("trading_pair", "") - - vol = float(perf.get("volume_traded", 0) or 0) - realized = float(perf.get("realized_pnl_quote", 0) or 0) - unrealized = float(perf.get("unrealized_pnl_quote", 0) or 0) - total_amount_quote = float(cfg.get("total_amount_quote", 0) or 0) - - controllers.append( - { - "id": ctrl_name, - "bot": bot_name, - "pair": trading_pair, - "vol": vol, - "realized": realized, - "unrealized": unrealized, - "net": realized + unrealized, - "total_amount_quote": total_amount_quote, - } - ) - - if not controllers: - return ["No controller data found\\."] - - # ---- Runtime ---- - earliest_dt = None - for bn in {c["bot"] for c in controllers}: - ts = bot_runs.get(bn) - if ts: - try: - ts_str = str(ts) - if ts_str.endswith("Z"): - ts_str = ts_str[:-1] + "+00:00" - dt = datetime.fromisoformat(ts_str) - if earliest_dt is None or dt < earliest_dt: - earliest_dt = dt - except (ValueError, TypeError): - pass - - now_dt = datetime.now(timezone.utc) - runtime_h = (now_dt - earliest_dt).total_seconds() / 3600 if earliest_dt else 0 - if earliest_dt: - start_str = earliest_dt.strftime("%b %d, %H:%M UTC") - else: - start_str = "N/A" - - # ---- Totals ---- - total_vol_q = sum(c["vol"] for c in controllers) - total_realized_q = sum(c["realized"] for c in controllers) - total_unrealized_q = sum(c["unrealized"] for c in controllers) - total_net_q = total_realized_q + total_unrealized_q - - to_usd = quote_usd_rate - total_vol = sum(_vol_to_usd(c, to_usd) for c in controllers) - total_realized = total_realized_q * to_usd - total_unrealized = total_unrealized_q * to_usd - total_net = total_net_q * to_usd - total_rebate_vol = sum( - _vol_to_usd(c, to_usd) - for c in controllers - if _pair_has_rebate(c["pair"], quote_currency) - ) - total_rebate = total_rebate_vol * REBATE_RATE - - vol_hr = total_vol / runtime_h if runtime_h > 0 else 0 - realized_hr = total_realized / runtime_h if runtime_h > 0 else 0 - rebate_hr = total_rebate / runtime_h if runtime_h > 0 else 0 - net_after_rebate = total_net + total_rebate - - # ---- Market volumes ---- - unique_pairs = {c["pair"] for c in controllers if c["pair"]} - market_lines = [] - for pair in sorted(unique_pairs): - vol_data = volumes_since.get(pair, (0, 0)) - vol_24h_base, vol_24h_quote = ( - vol_data if isinstance(vol_data, tuple) else (vol_data, 0) - ) - if vol_24h_base <= 0 and vol_24h_quote <= 0: - continue - mkt_usd = _market_vol_to_usd(pair, vol_24h_base, vol_24h_quote, to_usd, prices) - our_vol = sum(_vol_to_usd(c, to_usd) for c in controllers if c["pair"] == pair) - share = (our_vol / mkt_usd * 100) if mkt_usd > 0 else 0 - market_lines.append( - f" {_md_escape(pair)}: mkt \\${_md_escape(_fmt(mkt_usd))} " - f"\\| ours \\${_md_escape(_fmt(our_vol))} \\({_md_escape(f'{share:.2f}')}%\\)" - ) - - # ---- FX rate display ---- - qc = quote_currency.upper() - if qc not in ("USD", "USDT"): - fx_val = _fmt(1 / to_usd if to_usd > 0 else 0) - fx_display = f"1 USD \\= {_md_escape(fx_val)} {_md_escape(qc)}" - else: - fx_display = "" - - # ---- MSG 1: Overview ---- - num_bots = len({c["bot"] for c in controllers}) - num_ctrls = len(controllers) - - # ---- Balance summary ---- - balance_lines = [] - total_balance_usd = 0.0 - if balances: - token_totals: dict[str, float] = defaultdict(float) - token_values: dict[str, float] = defaultdict(float) - for account_data in balances.values(): - for connector_balances in account_data.values(): - if not isinstance(connector_balances, list): - continue - for b in connector_balances: - token = b.get("token", "") - value = float(b.get("value", 0) or 0) - units = float(b.get("units", 0) or 0) - if value > 0: - token_totals[token] += units - token_values[token] += value - total_balance_usd = sum(token_values.values()) - # Top tokens by value - sorted_tokens = sorted(token_values.items(), key=lambda x: x[1], reverse=True) - for token, value in sorted_tokens[:5]: - pct = (value / total_balance_usd * 100) if total_balance_usd > 0 else 0 - balance_lines.append( - f" {_md_escape(token)}: \\${_md_escape(_fmt(value))} \\({_md_escape(f'{pct:.0f}')}%\\)" - ) - - msg1_parts = [ - f"*📊 TRADING REPORT*", - f"━━━━━━━━━━━━━━━━━━━━━━━━", - f"", - f"⏱ {_md_escape(f'{runtime_h:.1f}')}h runtime \\(since {_md_escape(start_str)}\\)", - f"🤖 {num_bots} bots \\| {num_ctrls} controllers", - ] - if fx_display: - msg1_parts.append(f"💱 {fx_display}") - - if balance_lines: - msg1_parts += [ - f"", - f"💰 *Balance: \\${_md_escape(_fmt(total_balance_usd))}*", - ] + balance_lines - - msg1_parts += [ - f"", - f"{_emoji(total_net)} *Performance \\(USD\\)*", - f"Volume: \\${_md_escape(_fmt(total_vol))}", - f"Realized: {_md_escape(_sign(total_realized))}\\${_md_escape(_fmt(abs(total_realized)))}", - f"Unrealized: {_md_escape(_sign(total_unrealized))}\\${_md_escape(_fmt(abs(total_unrealized)))}", - f"Net PnL: {_md_escape(_sign(total_net))}\\${_md_escape(_fmt(abs(total_net)))}", - f"Rebate: \\+\\${_md_escape(_fmt(total_rebate))}", - f"Net\\+Rebate: {_md_escape(_sign(net_after_rebate))}\\${_md_escape(_fmt(abs(net_after_rebate)))}", - f"", - f"⏳ *Hourly Rates*", - f"Vol/hr: \\${_md_escape(_fmt(vol_hr))}", - f"Realized/hr: {_md_escape(_sign(realized_hr))}\\${_md_escape(_fmt(abs(realized_hr)))}", - f"Rebate/hr: \\+\\${_md_escape(_fmt(rebate_hr))}", - ] - - if market_lines: - msg1_parts += [f"", f"📈 *Market Share*"] + market_lines - - # ---- MSG 2: Controllers grouped by pair ---- - msg2_parts = [ - f"*📋 CONTROLLERS BY PAIR*", - f"━━━━━━━━━━━━━━━━━━━━━━━━", - ] - - pair_groups: dict[str, list[dict]] = defaultdict(list) - for c in controllers: - pair_groups[c["pair"] or "unknown"].append(c) - - sorted_pairs = sorted( - pair_groups.items(), key=lambda x: sum(c["vol"] for c in x[1]), reverse=True - ) - - for pair, ctrls in sorted_pairs: - grp_vol = sum(_vol_to_usd(c, to_usd) for c in ctrls) - grp_realized = sum(c["realized"] for c in ctrls) * to_usd - grp_unrealized = sum(c["unrealized"] for c in ctrls) * to_usd - grp_net = grp_realized + grp_unrealized - grp_rebate = ( - grp_vol * REBATE_RATE if _pair_has_rebate(pair, quote_currency) else 0 - ) - grp_adj = grp_net + grp_rebate - - pair_escaped = _md_escape(pair) - msg2_parts.append(f"") - msg2_parts.append( - f"▸ *{pair_escaped}* \\({len(ctrls)} ctrls\\) " - f"{_emoji(grp_adj)} {_md_escape(_sign(grp_adj))}\\${_md_escape(_fmt(abs(grp_adj)))}" - ) - - # Build controller table rows - sorted_ctrls = sorted(ctrls, key=lambda c: c["vol"], reverse=True) - - def _short_name(ctrl_id: str, pair: str) -> str: - name = ctrl_id - if name.startswith("mm_"): - name = name[3:] - # Strip all variations of pmm_mister prefix - name = re.sub(r"pmm_?mister_?", "", name) - pair_frag = pair.replace("-", "_") if pair else "" - if pair_frag: - name = name.replace(pair_frag, "").replace(pair, "") - while "__" in name: - name = name.replace("__", "_") - name = name.strip("_") - name = re.sub(r"config[_]?(\d+)", r"c\1", name) - return name or ctrl_id[:20] - - rows = [] - for c in sorted_ctrls: - vol_usd = _vol_to_usd(c, to_usd) - net_usd = c["net"] * to_usd - reb_usd = ( - vol_usd * REBATE_RATE if _pair_has_rebate(pair, quote_currency) else 0 - ) - adj_usd = net_usd + reb_usd - short = _short_name(c["id"], pair) - rows.append((short, vol_usd, net_usd, adj_usd)) - - # Calculate column widths - name_w = max(len(r[0]) for r in rows) - vol_strs = [f"${_fmt(r[1])}" for r in rows] - pnl_strs = [f"{_sign(r[2])}${_fmt(abs(r[2]))}" for r in rows] - adj_strs = [f"{_sign(r[3])}${_fmt(abs(r[3]))}" for r in rows] - - vol_w = max(len(s) for s in vol_strs) - pnl_w = max(len(s) for s in pnl_strs) - adj_w = max(len(s) for s in adj_strs) - - # Header + separator + rows inside a code block - hdr = ( - f"{'Name':<{name_w}} {'Vol':>{vol_w}} {'PnL':>{pnl_w}} {'Adj':>{adj_w}}" - ) - sep = "─" * len(hdr) - - table_lines = [hdr, sep] - for i, (short, vol_usd, net_usd, adj_usd) in enumerate(rows): - e = "+" if adj_usd >= 0 else "-" - table_lines.append( - f"{short:<{name_w}} {vol_strs[i]:>{vol_w}} {pnl_strs[i]:>{pnl_w}} {adj_strs[i]:>{adj_w}}" - ) - - # Pair subtotal row - sub_vol = f"${_fmt(grp_vol)}" - sub_pnl = f"{_sign(grp_net)}${_fmt(abs(grp_net))}" - sub_adj = f"{_sign(grp_adj)}${_fmt(abs(grp_adj))}" - table_lines.append(sep) - table_lines.append( - f"{'TOTAL':<{name_w}} {sub_vol:>{vol_w}} {sub_pnl:>{pnl_w}} {sub_adj:>{adj_w}}" - ) - - msg2_parts.append("```") - msg2_parts.extend(table_lines) - msg2_parts.append("```") - - return ["\n".join(msg1_parts), "\n".join(msg2_parts)] - - -# --------------------------------------------------------------------------- -# Telegram HTTP fallback (when context.bot is unavailable) -# --------------------------------------------------------------------------- - - -async def _send_telegram_http( - chat_id: int, text: str, parse_mode: str = "MarkdownV2" -) -> bool: - """Send a message via Telegram HTTP API directly. Used as fallback.""" - token = os.environ.get("TELEGRAM_BOT_TOKEN", "") - if not token: - logger.error("TELEGRAM_BOT_TOKEN not set, cannot send via HTTP fallback") - return False - try: - import httpx - - url = f"https://api.telegram.org/bot{token}/sendMessage" - payload = {"chat_id": chat_id, "text": text, "parse_mode": parse_mode} - async with httpx.AsyncClient(timeout=10) as http_client: - resp = await http_client.post(url, json=payload) - data = resp.json() - if data.get("ok"): - return True - # Retry without parse_mode if formatting fails - if "can't parse" in data.get("description", "").lower(): - payload.pop("parse_mode") - resp = await http_client.post(url, json=payload) - data = resp.json() - if data.get("ok"): - return True - logger.error(f"Telegram HTTP send failed: {data.get('description')}") - except Exception as e: - logger.error(f"Telegram HTTP fallback error: {e}") - return False - - -# --------------------------------------------------------------------------- -# Main entry point -# --------------------------------------------------------------------------- - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: - chat_id = context._chat_id if hasattr(context, "_chat_id") else None - if not chat_id: - return "No chat_id available" - - # Auto-resolve target_chat_id from notes if not explicitly set - send_to = config.target_chat_id - if not send_to: - try: - from condor.preferences import get_note - - user_data = context.user_data if hasattr(context, "user_data") else {} - # Look up note for the active server's group chat - server_name = ( - config.server_name - or get_config_manager().get_chat_default_server(chat_id) - ) - if server_name: - note_val = get_note(user_data, f"server.{server_name}.group_chat_id") - if note_val: - send_to = int(note_val) - except Exception: - pass - if not send_to: - # Fallback to CONDOR_CHAT_ID env var (the invoking user's chat) - env_chat = os.environ.get("CONDOR_CHAT_ID", "") - send_to = int(env_chat) if env_chat else chat_id - - # Connect to the specified server (default: brigado_2 where bots run) - if config.server_name: - client = await get_config_manager().get_client(config.server_name) - else: - client = await get_client(chat_id, context=context) - if not client: - return "Could not connect to API server" - - bots_data, bot_runs, balances = await asyncio.gather( - _get_bots_data(client), - _get_bot_runs(client), - _get_balances(client), - ) - - if not bots_data: - await context.bot.send_message(chat_id=send_to, text="No active bots found.") - return "No active bots" - - # Fetch controller configs for trading_pair - ctrl_configs = {} - for bot_name in bots_data: - try: - configs = await client.controllers.get_bot_controller_configs(bot_name) - if isinstance(configs, list): - for cfg in configs: - cid = cfg.get("id") or cfg.get("controller_id", "") - if cid: - ctrl_configs[cid] = cfg - except Exception: - pass - - # Determine trading pairs and connector - trading_pairs = set() - connector = "binance" - for bot_data in bots_data.values(): - if not isinstance(bot_data, dict): - continue - for ctrl_name, ctrl_data in bot_data.get("performance", {}).items(): - if not isinstance(ctrl_data, dict): - continue - cfg = ctrl_configs.get(ctrl_name, {}) - perf = ctrl_data.get("performance", ctrl_data) - pair = cfg.get("trading_pair", "") or perf.get("trading_pair", "") - if pair: - trading_pairs.add(pair) - conn = cfg.get("connector_name", "") or perf.get("connector_name", "") - if conn: - connector = conn.split("_")[0] - - # Build price pairs for USD conversion - price_pairs = list(trading_pairs) - for bt in {p.split("-")[0] for p in trading_pairs if "-" in p}: - if f"{bt}-USDT" not in price_pairs: - price_pairs.append(f"{bt}-USDT") - for qt in {p.split("-")[-1] for p in trading_pairs if "-" in p}: - if qt not in ("USDT", "USD") and f"USDT-{qt}" not in price_pairs: - price_pairs.append(f"USDT-{qt}") - - prices = await _get_prices(client, connector, price_pairs) - - # FX rate fallback - qc = config.quote_currency.upper() - fx_pair = f"USDT-{qc}" if qc not in ("USD", "USDT") else "" - if fx_pair and not prices.get(fx_pair): - fx_prices = await _get_prices(client, "binance", [fx_pair]) - prices.update(fx_prices) - - # Market volumes since deploy - earliest_dt = None - for bn in bots_data: - ts = bot_runs.get(bn) - if ts: - try: - ts_str = str(ts) - if ts_str.endswith("Z"): - ts_str = ts_str[:-1] + "+00:00" - dt = datetime.fromisoformat(ts_str) - if earliest_dt is None or dt < earliest_dt: - earliest_dt = dt - except (ValueError, TypeError): - pass - - volumes_since = {} - vol_tasks = { - pair: _get_market_volume_since(client, connector, pair, earliest_dt) - for pair in trading_pairs - } - vol_results = await asyncio.gather(*vol_tasks.values()) - for pair, vol_tuple in zip(vol_tasks.keys(), vol_results): - volumes_since[pair] = vol_tuple - - # Quote → USD rate - quote_usd_rate = 1.0 - if qc not in ("USD", "USDT"): - usdt_quote = prices.get(f"USDT-{qc}", 0) - if usdt_quote and usdt_quote > 0: - quote_usd_rate = 1.0 / usdt_quote - else: - quote_usdt = prices.get(f"{qc}-USDT", 0) - if quote_usdt and quote_usdt > 0: - quote_usd_rate = quote_usdt - - # Build and send - messages = _build_report( - bots_data, - bot_runs, - prices, - volumes_since, - quote_usd_rate, - quote_currency=qc, - ctrl_configs=ctrl_configs, - balances=balances, - ) - - sent = 0 - for i, msg in enumerate(messages): - if i > 0: - await asyncio.sleep(1) - for attempt in range(3): - try: - await context.bot.send_message( - chat_id=send_to, - text=msg, - parse_mode="MarkdownV2", - ) - sent += 1 - break - except Exception as e: - err = str(e).lower() - if "flood control" in err or "retry in" in err: - match = re.search(r"retry in (\d+)", err) - wait = int(match.group(1)) + 1 if match else 10 - await asyncio.sleep(wait) - else: - logger.error(f"Failed to send msg {i} via bot: {e}") - # Fallback: send via HTTP directly to Telegram API - if await _send_telegram_http(send_to, msg): - sent += 1 - break - - # Strip MarkdownV2 escaping for plain-text return - def _strip_md(text: str) -> str: - return text.replace("\\", "") - - plain_report = "\n\n".join(_strip_md(m) for m in messages) - - from condor.reports import ReportBuilder - - builder = ReportBuilder("Trading Report") - builder.source("routine", "bot_report").tags(["bots", "performance"]) - builder.manual_order() - - # ---- Parse controller data for charts ---- - controllers_parsed = [] - for bot_name, bot_data in bots_data.items(): - if isinstance(bot_data, str): - continue - performance = {} - if isinstance(bot_data, dict): - performance = bot_data.get("performance", {}) - elif isinstance(bot_data, list): - performance = { - f"ctrl_{i}": {"performance": c} for i, c in enumerate(bot_data) - } - for ctrl_name, ctrl_data in performance.items(): - if not isinstance(ctrl_data, dict): - continue - perf = ctrl_data.get("performance", ctrl_data) - cfg = ctrl_configs.get(ctrl_name, {}) - trading_pair = cfg.get("trading_pair", "") or perf.get("trading_pair", "") - vol = float(perf.get("volume_traded", 0) or 0) - realized = float(perf.get("realized_pnl_quote", 0) or 0) - unrealized = float(perf.get("unrealized_pnl_quote", 0) or 0) - total_amount_quote = float(cfg.get("total_amount_quote", 0) or 0) - controllers_parsed.append( - { - "id": ctrl_name, - "bot": bot_name, - "pair": trading_pair, - "vol": vol, - "realized": realized, - "unrealized": unrealized, - "net": realized + unrealized, - "total_amount_quote": total_amount_quote, - } - ) - - # ---- Compute totals for KPIs ---- - total_vol_usd = sum(_vol_to_usd(c, quote_usd_rate) for c in controllers_parsed) - total_realized_usd = sum(c["realized"] for c in controllers_parsed) * quote_usd_rate - total_unrealized_usd = ( - sum(c["unrealized"] for c in controllers_parsed) * quote_usd_rate - ) - total_net_usd = total_realized_usd + total_unrealized_usd - rebate_vol_usd = sum( - _vol_to_usd(c, quote_usd_rate) - for c in controllers_parsed - if _pair_has_rebate(c["pair"], qc) - ) - total_rebate_usd = rebate_vol_usd * REBATE_RATE - net_after_rebate_usd = total_net_usd + total_rebate_usd - num_bots = len({c["bot"] for c in controllers_parsed}) - num_ctrls = len(controllers_parsed) - - # Runtime - earliest_deploy = None - for bn in {c["bot"] for c in controllers_parsed}: - ts = bot_runs.get(bn) - if ts: - try: - ts_str = str(ts) - if ts_str.endswith("Z"): - ts_str = ts_str[:-1] + "+00:00" - dt = datetime.fromisoformat(ts_str) - if earliest_deploy is None or dt < earliest_deploy: - earliest_deploy = dt - except (ValueError, TypeError): - pass - now_utc = datetime.now(timezone.utc) - runtime_h = ( - (now_utc - earliest_deploy).total_seconds() / 3600 if earliest_deploy else 0 - ) - - # Balance - total_balance_usd = 0.0 - if balances: - for account_data in balances.values(): - for connector_balances in account_data.values(): - if not isinstance(connector_balances, list): - continue - for b in connector_balances: - total_balance_usd += float(b.get("value", 0) or 0) - - # ---- KPI cards ---- - pnl_trend = "up" if net_after_rebate_usd >= 0 else "down" - builder.kpi( - "Net PnL + Rebate", - f"{'+'if net_after_rebate_usd>=0 else ''}${_fmt(abs(net_after_rebate_usd))}", - delta=f"${_fmt(abs(net_after_rebate_usd / runtime_h) if runtime_h else 0)}/hr", - trend=pnl_trend, - ) - total_capital_usd = sum( - _amount_quote_to_usd(c, quote_usd_rate) for c in controllers_parsed - ) - builder.kpi( - "Volume", - f"${_fmt(total_vol_usd)}", - delta=f"${_fmt(total_vol_usd / runtime_h if runtime_h else 0)}/hr", - ) - builder.kpi( - "Rebate", - f"${_fmt(total_rebate_usd)}", - delta=f"${_fmt(total_rebate_usd / runtime_h if runtime_h else 0)}/hr", - trend="up", - ) - if total_capital_usd > 0: - builder.kpi( - "Capital Deployed", - f"${_fmt(total_capital_usd)}", - delta=f"{num_ctrls} controllers", - ) - builder.kpi("Balance", f"${_fmt(total_balance_usd)}") - builder.kpi( - "Runtime", - f"{runtime_h:.1f}h", - delta=f"{num_bots} bots, {num_ctrls} controllers", - ) - - # ---- Charts ---- - if controllers_parsed: - builder.plotly( - _build_pair_overview_chart( - controllers_parsed, quote_usd_rate, volumes_since, prices, qc - ) - ) - builder.plotly( - _build_pnl_chart(controllers_parsed, quote_usd_rate, qc, ctrl_configs) - ) - - # ---- Controller details table ---- - ctrl_rows = [] - for c in sorted( - controllers_parsed, key=lambda x: x["net"] * quote_usd_rate, reverse=True - ): - vol_usd = _vol_to_usd(c, quote_usd_rate) - net_usd = c["net"] * quote_usd_rate - reb_usd = vol_usd * REBATE_RATE if _pair_has_rebate(c["pair"], qc) else 0 - adj_usd = net_usd + reb_usd - cap_usd = _amount_quote_to_usd(c, quote_usd_rate) - ctrl_rows.append( - { - "Controller": _short_chart_name(c["id"], c["pair"]), - "Pair": c["pair"], - "Capital": f"${_fmt(cap_usd)}" if cap_usd > 0 else "-", - "Volume": f"${_fmt(vol_usd)}", - "Realized": f"{'+'if c['realized']*quote_usd_rate>=0 else '-'}${_fmt(abs(c['realized']*quote_usd_rate))}", - "Unrealized": f"{'+'if c['unrealized']*quote_usd_rate>=0 else '-'}${_fmt(abs(c['unrealized']*quote_usd_rate))}", - "Rebate": f"+${_fmt(reb_usd)}", - "Total": f"{'+'if adj_usd>=0 else '-'}${_fmt(abs(adj_usd))}", - } - ) - if ctrl_rows: - builder.table(ctrl_rows) - - # ---- Pair summary table ---- - pair_groups: dict[str, list[dict]] = defaultdict(list) - for c in controllers_parsed: - pair_groups[c["pair"] or "unknown"].append(c) - pair_rows = [] - for pair, ctrls in sorted( - pair_groups.items(), key=lambda x: sum(c["vol"] for c in x[1]), reverse=True - ): - grp_vol = sum(_vol_to_usd(c, quote_usd_rate) for c in ctrls) - grp_realized = sum(c["realized"] for c in ctrls) * quote_usd_rate - grp_unrealized = sum(c["unrealized"] for c in ctrls) * quote_usd_rate - grp_net = grp_realized + grp_unrealized - grp_rebate = grp_vol * REBATE_RATE if _pair_has_rebate(pair, qc) else 0 - grp_adj = grp_net + grp_rebate - # Market share - vol_data = volumes_since.get(pair, (0, 0)) - vol_base, vol_quote = vol_data if isinstance(vol_data, tuple) else (vol_data, 0) - mkt_usd = _market_vol_to_usd(pair, vol_base, vol_quote, quote_usd_rate, prices) - share = f"{grp_vol / mkt_usd * 100:.2f}%" if mkt_usd > 0 else "N/A" - grp_capital = sum(_amount_quote_to_usd(c, quote_usd_rate) for c in ctrls) - pair_rows.append( - { - "Pair": pair, - "Controllers": str(len(ctrls)), - "Capital": f"${_fmt(grp_capital)}" if grp_capital > 0 else "-", - "Volume": f"${_fmt(grp_vol)}", - "Realized": f"{'+'if grp_realized>=0 else '-'}${_fmt(abs(grp_realized))}", - "Net PnL": f"{'+'if grp_net>=0 else '-'}${_fmt(abs(grp_net))}", - "Rebate": f"+${_fmt(grp_rebate)}", - "Adjusted": f"{'+'if grp_adj>=0 else '-'}${_fmt(abs(grp_adj))}", - "Mkt Share": share, - } - ) - if pair_rows: - builder.table(pair_rows) - - # ---- Balance table ---- - if balances: - token_totals: dict[str, float] = defaultdict(float) - token_values: dict[str, float] = defaultdict(float) - for account_data in balances.values(): - for connector_balances in account_data.values(): - if not isinstance(connector_balances, list): - continue - for b in connector_balances: - token = b.get("token", "") - value = float(b.get("value", 0) or 0) - units = float(b.get("units", 0) or 0) - if value > 0: - token_totals[token] += units - token_values[token] += value - if token_values: - bal_rows = [] - for token, value in sorted( - token_values.items(), key=lambda x: x[1], reverse=True - ): - pct = (value / total_balance_usd * 100) if total_balance_usd > 0 else 0 - bal_rows.append( - { - "Token": token, - "Amount": _fmt(token_totals[token], 4), - "Value (USD)": f"${_fmt(value)}", - "Weight": f"{pct:.1f}%", - } - ) - builder.table(bal_rows) - - # ---- Config Analysis section ---- - if len(ctrl_configs) > 1: - all_params = set() - for cfg in ctrl_configs.values(): - all_params.update(cfg.keys()) - all_params -= _CONFIG_SKIP_PARAMS - sorted_params = sorted( - all_params, key=lambda p: (0 if p in _CONFIG_KEY_PARAMS else 1, p) - ) - - # Group configs by pair - cfg_pair_groups: dict[str, dict] = defaultdict(dict) - for cid, cfg in ctrl_configs.items(): - pair = cfg.get("trading_pair", "unknown") - cfg_pair_groups[pair][cid] = cfg - - # Count total diffs - total_diffs = 0 - pair_sections = [] - for pair, pair_cfgs in sorted( - cfg_pair_groups.items(), key=lambda x: -len(x[1]) - ): - if len(pair_cfgs) < 2: - continue - labels = list(pair_cfgs.keys()) - shorts = [_short_chart_name(l, pair) for l in labels] - - diff_rows = [] - for param in sorted_params: - values = [_cfg_normalize(pair_cfgs[l].get(param)) for l in labels] - non_none = [v for v in values if v is not None] - if len(non_none) < 2 or len(set(str(v) for v in non_none)) <= 1: - continue - is_key = param in _CONFIG_KEY_PARAMS - row = {"Parameter": f"{'🔴 ' if is_key else '🟡 '}{param}"} - for short, label in zip(shorts, labels): - row[short] = _cfg_fmt_value(pair_cfgs[label].get(param)) - diff_rows.append(row) - - if diff_rows: - total_diffs += len(diff_rows) - pair_sections.append((pair, len(pair_cfgs), diff_rows)) - - if pair_sections: - builder.markdown( - "## Config Analysis\n\nParameter differences across running controllers. 🔴 = key tuning param, 🟡 = other." - ) - for pair, n_cfgs, rows in pair_sections: - builder.markdown( - f"### {pair} — {len(rows)} diffs across {n_cfgs} controllers" - ) - builder.table(rows) - - # Config charts - labeled_configs = {f"\U0001f7e2 {k}": v for k, v in ctrl_configs.items()} - running_ids = set(ctrl_configs.keys()) - - fig_pc = _build_config_parallel_coords(labeled_configs, running_ids) - if fig_pc: - if not pair_sections: - builder.markdown("## Config Analysis") - builder.plotly(fig_pc) - - fig_spread = _build_config_spread_chart(labeled_configs) - if fig_spread: - builder.plotly(fig_spread) - - await builder.save() - - return f"Report sent: {sent}/{len(messages)} messages\n\n{plain_report}" diff --git a/agents/brigado/routines/brl_dashboard.py b/agents/brigado/routines/brl_dashboard.py deleted file mode 100644 index 691844004..000000000 --- a/agents/brigado/routines/brl_dashboard.py +++ /dev/null @@ -1,573 +0,0 @@ -"""Multi-timeframe USDT-BRL technical dashboard with Plotly chart.""" - -CATEGORY = "Market Data" - -import asyncio -import io -import logging -from datetime import datetime, timezone - -import numpy as np -import pandas as pd -import plotly.graph_objects as go -from plotly.subplots import make_subplots -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from config_manager import get_client -from routines.base import RoutineResult - -logger = logging.getLogger(__name__) - - -class Config(BaseModel): - """USDT-BRL multi-timeframe dashboard: intraday, weekly, monthly, quarterly.""" - - trading_pair: str = Field(default="USDT-BRL", description="Trading pair") - connector_name: str = Field(default="binance", description="Exchange connector") - - -# --------------------------------------------------------------------------- -# Indicators -# --------------------------------------------------------------------------- - - -def compute_sma(series: pd.Series, period: int) -> pd.Series: - return series.rolling(window=period, min_periods=period).mean() - - -def compute_ema(series: pd.Series, period: int) -> pd.Series: - return series.ewm(span=period, adjust=False).mean() - - -def compute_rsi(series: pd.Series, period: int = 14) -> pd.Series: - delta = series.diff() - gain = delta.where(delta > 0, 0.0) - loss = (-delta).where(delta < 0, 0.0) - avg_gain = gain.ewm(alpha=1 / period, min_periods=period).mean() - avg_loss = loss.ewm(alpha=1 / period, min_periods=period).mean() - rs = avg_gain / avg_loss.replace(0, np.nan) - rsi = 100 - (100 / (1 + rs)) - return rsi.fillna(50) - - -def find_sr_levels( - highs: pd.Series, lows: pd.Series, window: int = 20 -) -> tuple[list[float], list[float]]: - """Find support/resistance using rolling min/max pivots.""" - n = len(highs) - if n < window * 2: - return [], [] - - h = highs.values - l = lows.values - close = float(highs.iloc[-1] + lows.iloc[-1]) / 2 - - levels: dict[float, int] = {} - for w in (window, window * 2): - if w >= n // 2: - continue - for i in range(w, n - w): - if l[i] == min(l[i - w : i + w + 1]): - lv = round(float(l[i]), 4) - levels[lv] = levels.get(lv, 0) + 1 - if h[i] == max(h[i - w : i + w + 1]): - lv = round(float(h[i]), 4) - levels[lv] = levels.get(lv, 0) + 1 - - if not levels: - return [], [] - - # Cluster nearby levels (within 0.3%) - sorted_levels = sorted(levels.items()) - clusters: list[list[tuple[float, int]]] = [[sorted_levels[0]]] - for price, hits in sorted_levels[1:]: - cluster_avg = np.mean([p for p, _ in clusters[-1]]) - if abs(price - cluster_avg) / cluster_avg < 0.003: - clusters[-1].append((price, hits)) - else: - clusters.append([(price, hits)]) - - merged = [] - for cluster in clusters: - total_hits = sum(h for _, h in cluster) - avg_price = np.average([p for p, _ in cluster], weights=[h for _, h in cluster]) - merged.append((round(float(avg_price), 4), total_hits)) - - supports = sorted([lv for lv, _ in merged if lv < close], reverse=True)[:3] - resistances = sorted([lv for lv, _ in merged if lv >= close])[:3] - return supports, resistances - - -# --------------------------------------------------------------------------- -# Data fetching & processing -# --------------------------------------------------------------------------- - -TIMEFRAMES = [ - {"label": "Intraday (1m)", "interval": "1m", "days": 1, "short_label": "1D"}, - {"label": "Semanal (15m)", "interval": "15m", "days": 7, "short_label": "1W"}, - {"label": "Mensual (1h)", "interval": "1h", "days": 30, "short_label": "1M"}, - {"label": "Trimestral (1d)", "interval": "1d", "days": 90, "short_label": "3M"}, -] - - -async def fetch_candles( - client, connector: str, pair: str, interval: str, days: int -) -> pd.DataFrame: - """Fetch candles and return as DataFrame.""" - result = await client.market_data.get_candles_last_days( - connector, pair, days=days, interval=interval - ) - # API returns a list directly, or a dict with "candles"/"data" key - if isinstance(result, list): - records = result - elif isinstance(result, dict): - records = result.get("data", result.get("candles", [])) - else: - records = [] - if not records: - return pd.DataFrame() - - df = pd.DataFrame(records) - for col in ("open", "high", "low", "close", "volume"): - df[col] = pd.to_numeric(df[col], errors="coerce") - - ts_col = df["timestamp"] - unit = "s" if ts_col.iloc[0] > 1e12 / 1000 else "ms" - df["datetime"] = pd.to_datetime(ts_col, unit=unit, utc=True) - df = df.sort_values("datetime").reset_index(drop=True) - return df - - -def compute_indicators(df: pd.DataFrame) -> dict: - """Compute all indicators for a timeframe.""" - close = df["close"] - sma20 = compute_sma(close, 20) - sma50 = compute_sma(close, 50) - ema9 = compute_ema(close, 9) - ema21 = compute_ema(close, 21) - rsi = compute_rsi(close, 14) - - current = float(close.iloc[-1]) - open_price = float(close.iloc[0]) - change_pct = (current - open_price) / open_price * 100 - - high = float(df["high"].max()) - low = float(df["low"].min()) - - supports, resistances = find_sr_levels(df["high"], df["low"]) - - # Trend direction - sma20_now = sma20.iloc[-1] - sma50_now = sma50.iloc[-1] - if pd.notna(sma20_now) and pd.notna(sma50_now): - if current > sma20_now > sma50_now: - trend = "ALCISTA" - elif current < sma20_now < sma50_now: - trend = "BAJISTA" - else: - trend = "LATERAL" - else: - trend = "N/A" - - return { - "sma20": sma20, - "sma50": sma50, - "ema9": ema9, - "ema21": ema21, - "rsi": rsi, - "current": current, - "change_pct": change_pct, - "high": high, - "low": low, - "supports": supports, - "resistances": resistances, - "trend": trend, - "rsi_now": float(rsi.iloc[-1]) if pd.notna(rsi.iloc[-1]) else 50, - "sma20_now": float(sma20_now) if pd.notna(sma20_now) else current, - "sma50_now": float(sma50_now) if pd.notna(sma50_now) else current, - } - - -# --------------------------------------------------------------------------- -# Chart -# --------------------------------------------------------------------------- - -COLORS = { - "bg_paper": "#0d1117", - "bg_plot": "#161b22", - "grid": "#21262d", - "text": "#c9d1d9", - "text_dim": "#8b949e", - "green": "#3fb950", - "red": "#f85149", - "sma20": "#f0883e", - "sma50": "#58a6ff", - "rsi_line": "#bc8cff", - "support": "#3fb950", - "resistance": "#f85149", -} - - -def build_chart(timeframe_data: list[dict], pair: str) -> io.BytesIO: - """Build a 4x2 dashboard: candlestick+SMA (left col), RSI (right col) per timeframe.""" - fig = make_subplots( - rows=4, - cols=2, - column_widths=[0.78, 0.22], - row_heights=[0.25, 0.25, 0.25, 0.25], - vertical_spacing=0.045, - horizontal_spacing=0.03, - subplot_titles=( - [ - item["label"] - for item in timeframe_data - for _ in range(2) # one title per subplot pair - ] - if False - else None - ), # We'll add custom titles - ) - - current_price = timeframe_data[0]["indicators"]["current"] if timeframe_data else 0 - - for i, item in enumerate(timeframe_data): - row = i + 1 - df = item["df"] - ind = item["indicators"] - tf = item["tf"] - dates = df["datetime"] - - # --- Candlestick --- - fig.add_trace( - go.Candlestick( - x=dates, - open=df["open"], - high=df["high"], - low=df["low"], - close=df["close"], - increasing_line_color=COLORS["green"], - decreasing_line_color=COLORS["red"], - increasing_fillcolor=COLORS["green"], - decreasing_fillcolor=COLORS["red"], - showlegend=False, - name=tf["short_label"], - ), - row=row, - col=1, - ) - - # SMA20 - sma20_valid = ind["sma20"].dropna() - if len(sma20_valid) > 0: - fig.add_trace( - go.Scatter( - x=dates.iloc[sma20_valid.index], - y=sma20_valid, - line=dict(color=COLORS["sma20"], width=1.2), - showlegend=(i == 0), - name="SMA 20", - ), - row=row, - col=1, - ) - - # SMA50 - sma50_valid = ind["sma50"].dropna() - if len(sma50_valid) > 0: - fig.add_trace( - go.Scatter( - x=dates.iloc[sma50_valid.index], - y=sma50_valid, - line=dict(color=COLORS["sma50"], width=1.2), - showlegend=(i == 0), - name="SMA 50", - ), - row=row, - col=1, - ) - - # Support lines - t0, t1 = dates.iloc[0], dates.iloc[-1] - for s in ind["supports"]: - fig.add_shape( - type="line", - x0=t0, - x1=t1, - y0=s, - y1=s, - line=dict(color=COLORS["support"], width=0.7, dash="dot"), - opacity=0.5, - row=row, - col=1, - ) - - # Resistance lines - for r in ind["resistances"]: - fig.add_shape( - type="line", - x0=t0, - x1=t1, - y0=r, - y1=r, - line=dict(color=COLORS["resistance"], width=0.7, dash="dot"), - opacity=0.5, - row=row, - col=1, - ) - - # --- RSI panel --- - rsi_valid = ind["rsi"].dropna() - if len(rsi_valid) > 0: - fig.add_trace( - go.Scatter( - x=dates.iloc[rsi_valid.index], - y=rsi_valid, - line=dict(color=COLORS["rsi_line"], width=1.5), - fill="tozeroy", - fillcolor="rgba(188,140,255,0.08)", - showlegend=False, - ), - row=row, - col=2, - ) - - # Overbought/oversold bands - fig.add_hline( - y=70, - line_dash="dot", - line_color=COLORS["red"], - opacity=0.4, - row=row, - col=2, - ) - fig.add_hline( - y=30, - line_dash="dot", - line_color=COLORS["green"], - opacity=0.4, - row=row, - col=2, - ) - - # --- Row label annotation --- - trend_color = COLORS["green"] if ind["change_pct"] >= 0 else COLORS["red"] - arrow = "+" if ind["change_pct"] >= 0 else "" - label_text = ( - f"{tf['short_label']} {tf['label'].split('(')[1].rstrip(')')}" - f" | {ind['current']:.3f}" - f" | {arrow}{ind['change_pct']:.2f}%" - f" | RSI: {ind['rsi_now']:.0f}" - f" | {ind['trend']}" - ) - fig.add_annotation( - text=label_text, - xref="paper", - yref="paper", - x=0, - y=1 - (i * 0.25) + 0.005, - showarrow=False, - font=dict(size=11, color=COLORS["text"]), - xanchor="left", - yanchor="bottom", - ) - - # --- Header --- - # Get quarterly change - q_change = ( - timeframe_data[3]["indicators"]["change_pct"] if len(timeframe_data) > 3 else 0 - ) - m_change = ( - timeframe_data[2]["indicators"]["change_pct"] if len(timeframe_data) > 2 else 0 - ) - w_change = ( - timeframe_data[1]["indicators"]["change_pct"] if len(timeframe_data) > 1 else 0 - ) - d_change = timeframe_data[0]["indicators"]["change_pct"] if timeframe_data else 0 - - title_text = ( - f"{pair} | {current_price:.3f}" - f" | D: {d_change:+.2f}% W: {w_change:+.2f}% M: {m_change:+.2f}% 3M: {q_change:+.2f}%" - ) - - # --- Layout --- - fig.update_layout( - title=dict(text=title_text, font=dict(color=COLORS["text"], size=14), x=0.5), - paper_bgcolor=COLORS["bg_paper"], - plot_bgcolor=COLORS["bg_plot"], - font=dict(color=COLORS["text_dim"], size=9), - legend=dict( - orientation="h", - yanchor="bottom", - y=1.05, - xanchor="center", - x=0.4, - font=dict(size=10, color=COLORS["text"]), - bgcolor="rgba(0,0,0,0)", - ), - margin=dict(l=55, r=30, t=100, b=30), - width=1300, - height=1400, - xaxis_rangeslider_visible=False, - xaxis3_rangeslider_visible=False, - xaxis5_rangeslider_visible=False, - xaxis7_rangeslider_visible=False, - ) - - # Style axes - for row in range(1, 5): - # Price axis - fig.update_yaxes( - gridcolor=COLORS["grid"], - zerolinecolor=COLORS["grid"], - tickformat=".3f", - row=row, - col=1, - ) - fig.update_xaxes( - gridcolor=COLORS["grid"], showticklabels=(row == 4), row=row, col=1 - ) - # RSI axis - fig.update_yaxes( - gridcolor=COLORS["grid"], - range=[0, 100], - tickvals=[30, 50, 70], - row=row, - col=2, - ) - fig.update_xaxes(gridcolor=COLORS["grid"], showticklabels=False, row=row, col=2) - - buf = io.BytesIO() - fig.write_image(buf, format="png", scale=2) - buf.seek(0) - return buf, fig - - -# --------------------------------------------------------------------------- -# Text summary -# --------------------------------------------------------------------------- - - -def format_summary(timeframe_data: list[dict], pair: str) -> str: - lines = [f"Dashboard {pair}"] - for item in timeframe_data: - ind = item["indicators"] - tf = item["tf"] - arrow = "+" if ind["change_pct"] >= 0 else "" - sup_str = ", ".join(f"{s:.3f}" for s in ind["supports"][:2]) or "N/A" - res_str = ", ".join(f"{r:.3f}" for r in ind["resistances"][:2]) or "N/A" - lines.append( - f"\n{tf['label']}:" - f"\n Precio: {ind['current']:.3f} ({arrow}{ind['change_pct']:.2f}%)" - f"\n Tendencia: {ind['trend']} | RSI: {ind['rsi_now']:.0f}" - f"\n SMA20: {ind['sma20_now']:.3f} | SMA50: {ind['sma50_now']:.3f}" - f"\n Soporte: {sup_str} | Resistencia: {res_str}" - f"\n Rango: {ind['low']:.3f} - {ind['high']:.3f}" - ) - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# Telegram photo fallback -# --------------------------------------------------------------------------- - - -async def _send_photo_http(chat_id: int, buf: io.BytesIO, caption: str) -> None: - import os - - import httpx - - token = os.environ.get("TELEGRAM_BOT_TOKEN") or os.environ.get("TELEGRAM_TOKEN", "") - if not token: - return - - url = f"https://api.telegram.org/bot{token}/sendPhoto" - buf.seek(0) - async with httpx.AsyncClient(timeout=30) as http_client: - resp = await http_client.post( - url, - data={"chat_id": chat_id, "caption": caption}, - files={"photo": ("dashboard.png", buf, "image/png")}, - ) - if not resp.json().get("ok"): - logger.warning(f"sendPhoto failed: {resp.text}") - - -# --------------------------------------------------------------------------- -# Main -# --------------------------------------------------------------------------- - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: - chat_id = context._chat_id if hasattr(context, "_chat_id") else None - client = await get_client(chat_id, context=context) - if not client: - return "No server available" - - # Fetch all timeframes in parallel - fetch_tasks = [ - fetch_candles( - client, - config.connector_name, - config.trading_pair, - tf["interval"], - tf["days"], - ) - for tf in TIMEFRAMES - ] - results = await asyncio.gather(*fetch_tasks, return_exceptions=True) - - timeframe_data = [] - for tf, result in zip(TIMEFRAMES, results): - if isinstance(result, Exception): - logger.warning(f"Failed to fetch {tf['label']}: {result}") - continue - if result.empty or len(result) < 10: - logger.warning(f"Not enough data for {tf['label']}: {len(result)} candles") - continue - - indicators = compute_indicators(result) - timeframe_data.append( - { - "tf": tf, - "label": tf["label"], - "df": result, - "indicators": indicators, - } - ) - - if not timeframe_data: - return "No candle data available for any timeframe" - - # Generate chart - chart_bytes: bytes | None = None - plotly_fig = None - try: - buf, plotly_fig = build_chart(timeframe_data, config.trading_pair) - chart_bytes = buf.getvalue() - caption = f"{config.trading_pair} Multi-Timeframe Dashboard" - if chat_id: - buf.seek(0) - if context.bot: - await context.bot.send_photo( - chat_id=chat_id, photo=buf, caption=caption - ) - else: - await _send_photo_http(chat_id, buf, caption) - except Exception as e: - logger.warning(f"Chart generation/send failed: {e}") - - text = format_summary(timeframe_data, config.trading_pair) - - from condor.reports import ReportBuilder - - builder = ReportBuilder(f"Dashboard: {config.trading_pair}") - builder.source("routine", "brl_dashboard").tags(["dashboard", config.trading_pair]) - builder.markdown(text) - if plotly_fig is not None: - builder.plotly(plotly_fig) - await builder.save() - - if context.bot and chart_bytes: - return RoutineResult(text=text, chart_image=chart_bytes) - return text diff --git a/agents/brigado/routines/btc_brl_arbitrage.py b/agents/brigado/routines/btc_brl_arbitrage.py deleted file mode 100644 index a51e3ccd1..000000000 --- a/agents/brigado/routines/btc_brl_arbitrage.py +++ /dev/null @@ -1,608 +0,0 @@ -"""BTC-BRL arbitrage: spread analysis + quoting strategy simulation.""" - -CATEGORY = "Market Data" - -import asyncio -import io -import logging - -import numpy as np -import pandas as pd -import plotly.graph_objects as go -from plotly.subplots import make_subplots -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from config_manager import get_client -from routines.base import RoutineResult - -logger = logging.getLogger(__name__) - -COLORS = { - "bg_paper": "#0d1117", - "bg_plot": "#161b22", - "grid": "#21262d", - "text": "#c9d1d9", - "text_dim": "#8b949e", - "usdt": "#f0b90b", - "brl": "#3fb950", - "spread": "#bc8cff", - "long": "#3fb950", - "short": "#f85149", - "pnl": "#58a6ff", - "fees": "#f78166", -} - -LAYOUT_BASE = dict( - paper_bgcolor=COLORS["bg_paper"], - plot_bgcolor=COLORS["bg_plot"], - font=dict(color=COLORS["text_dim"], size=10), - legend=dict( - orientation="h", - yanchor="bottom", - y=1.05, - xanchor="center", - x=0.5, - font=dict(size=10, color=COLORS["text"]), - bgcolor="rgba(0,0,0,0)", - ), - margin=dict(l=60, r=30, t=100, b=40), -) - - -class Config(BaseModel): - """BTC-BRL arbitrage: spread analysis + quoting strategy simulation.""" - - connector_name: str = Field(default="binance", description="Exchange connector") - lookback_seconds: int = Field( - default=3600, description="1s candles to fetch (3600=1h)" - ) - trade_size_usdt: float = Field( - default=100.0, description="Notional per trade in USDT" - ) - fee_bps: float = Field( - default=10.0, description="Round-trip fee in bps (maker+taker)" - ) - min_spread_bps: float = Field(default=2.0, description="Min spread to enter (bps)") - max_position_usdt: float = Field( - default=5000.0, description="Max accumulated position (USDT)" - ) - - -async def _fetch_candles( - client, connector: str, pair: str, interval: str, max_records: int -) -> pd.DataFrame: - result = await client.market_data.get_candles( - connector, pair, interval=interval, max_records=max_records - ) - if isinstance(result, list): - records = result - elif isinstance(result, dict): - records = result.get("data", result.get("candles", [])) - else: - records = [] - if not records: - return pd.DataFrame() - df = pd.DataFrame(records) - for col in ("open", "high", "low", "close", "volume", "quote_asset_volume"): - if col in df.columns: - df[col] = pd.to_numeric(df[col], errors="coerce") - ts = df["timestamp"] - unit = "s" if ts.iloc[0] > 1e12 / 1000 else "ms" - df["datetime"] = pd.to_datetime(ts, unit=unit, utc=True) - return df.sort_values("datetime").reset_index(drop=True) - - -def _style_axes(fig, rows): - for row in range(1, rows + 1): - fig.update_yaxes( - gridcolor=COLORS["grid"], zerolinecolor=COLORS["grid"], row=row, col=1 - ) - fig.update_xaxes(gridcolor=COLORS["grid"], row=row, col=1) - - -def _merge_spread( - price_usdt: pd.DataFrame, price_brl: pd.DataFrame, usdtbrl: pd.DataFrame -) -> pd.DataFrame: - """Merge 1s candles and compute spread %.""" - for df in (price_usdt, price_brl, usdtbrl): - df["ts"] = df["datetime"].dt.floor("s") - - merged = pd.merge( - price_usdt[["ts", "close"]].rename(columns={"close": "btc_usdt"}), - price_brl[["ts", "close"]].rename(columns={"close": "btc_brl"}), - on="ts", - how="inner", - ) - merged = pd.merge( - merged, - usdtbrl[["ts", "close"]].rename(columns={"close": "usdt_brl_rate"}), - on="ts", - how="inner", - ) - merged["btc_brl_usd"] = merged["btc_brl"] / merged["usdt_brl_rate"] - merged["spread_pct"] = ( - (merged["btc_brl_usd"] - merged["btc_usdt"]) / merged["btc_usdt"] - ) * 100 - merged["spread_bps"] = merged["spread_pct"] * 100 - return merged.sort_values("ts").reset_index(drop=True) - - -def simulate_quoting( - merged: pd.DataFrame, - trade_size_usdt: float, - fee_bps: float, - min_spread_bps: float, - max_position_usdt: float, -) -> pd.DataFrame: - """Simulate a one-sided quoting strategy that captures spread. - - Logic: - - When BRL premium (spread > min_spread): sell on BRL side (capture premium) - → accumulate short BRL / long USDT exposure - - When USDT premium (spread < -min_spread): sell on USDT side (capture premium) - → accumulate short USDT / long BRL exposure - - Position flips naturally as spread oscillates - - Fee deducted on each trade - """ - n = len(merged) - fee_pct = fee_bps / 10000 - - # Track state - position_usdt = np.zeros( - n - ) # net position in USDT terms (+ = long BRL, - = short BRL) - cum_captured = np.zeros(n) # cumulative spread captured (before fees) - cum_fees = np.zeros(n) # cumulative fees paid - cum_pnl = np.zeros(n) # net PnL (captured - fees) - trade_count = np.zeros(n, dtype=int) - actions = [""] * n - - pos = 0.0 - captured = 0.0 - fees = 0.0 - trades = 0 - - for i in range(n): - spread_bps = merged["spread_bps"].iloc[i] - abs_spread = abs(spread_bps) - - if abs_spread > min_spread_bps: - # Check position limits - if spread_bps > 0 and pos > -max_position_usdt: - # BRL premium → sell on BRL (go short BRL exposure) - trade_pnl = (abs_spread / 10000) * trade_size_usdt - trade_fee = fee_pct * trade_size_usdt - captured += trade_pnl - fees += trade_fee - pos -= trade_size_usdt - trades += 1 - actions[i] = "sell_brl" - elif spread_bps < 0 and pos < max_position_usdt: - # USDT premium → sell on USDT (go long BRL exposure) - trade_pnl = (abs_spread / 10000) * trade_size_usdt - trade_fee = fee_pct * trade_size_usdt - captured += trade_pnl - fees += trade_fee - pos += trade_size_usdt - trades += 1 - actions[i] = "sell_usdt" - - position_usdt[i] = pos - cum_captured[i] = captured - cum_fees[i] = fees - cum_pnl[i] = captured - fees - trade_count[i] = trades - - merged = merged.copy() - merged["position_usdt"] = position_usdt - merged["cum_captured"] = cum_captured - merged["cum_fees"] = cum_fees - merged["cum_pnl"] = cum_pnl - merged["trade_count"] = trade_count - merged["action"] = actions - - # Mark-to-market: unrealized PnL from position * current spread change - # Position was entered at various spreads; approximate with running avg - merged["unrealized"] = 0.0 - avg_entry_spread = 0.0 - total_pos = 0.0 - for i in range(n): - if actions[i]: - entry_spread = merged["spread_bps"].iloc[i] - trade_pos = ( - trade_size_usdt if actions[i] == "sell_usdt" else -trade_size_usdt - ) - if abs(total_pos + trade_pos) > 0.01: - avg_entry_spread = ( - avg_entry_spread * total_pos + entry_spread * trade_pos - ) / (total_pos + trade_pos) - total_pos += trade_pos - current_spread = merged["spread_bps"].iloc[i] - # If short BRL (pos < 0), we profit when spread decreases - # If long BRL (pos > 0), we profit when spread increases - spread_change = current_spread - avg_entry_spread - merged.loc[merged.index[i], "unrealized"] = (spread_change / 10000) * total_pos - - merged["total_pnl"] = merged["cum_pnl"] + merged["unrealized"] - - return merged - - -def build_spread_chart(merged: pd.DataFrame) -> go.Figure: - """Chart 1: Raw spread analysis.""" - avg_spread = merged["spread_pct"].mean() - curr_spread = merged["spread_pct"].iloc[-1] - std_spread = merged["spread_pct"].std() - max_spread = merged["spread_pct"].max() - min_spread = merged["spread_pct"].min() - current_btc = float(merged["btc_usdt"].iloc[-1]) - - fig = make_subplots( - rows=2, - cols=1, - row_heights=[0.55, 0.45], - vertical_spacing=0.08, - subplot_titles=[ - "BTC: USDT vs BRL→USD (1s)", - f"Spread % — avg {avg_spread:+.4f}%, std {std_spread:.4f}%", - ], - ) - - fig.add_trace( - go.Scatter( - x=merged["ts"], - y=merged["btc_usdt"], - line=dict(color=COLORS["usdt"], width=1.5), - name="BTC-USDT", - ), - row=1, - col=1, - ) - fig.add_trace( - go.Scatter( - x=merged["ts"], - y=merged["btc_brl_usd"], - line=dict(color=COLORS["brl"], width=1.5), - name="BTC-BRL (USD)", - ), - row=1, - col=1, - ) - - # Spread with color fill - fig.add_trace( - go.Scatter( - x=merged["ts"], - y=merged["spread_pct"], - line=dict(color=COLORS["spread"], width=1.2), - fill="tozeroy", - fillcolor="rgba(188,140,255,0.08)", - name="Spread %", - ), - row=2, - col=1, - ) - fig.add_hline(y=0, line_color=COLORS["grid"], row=2, col=1) - fig.add_hline( - y=avg_spread, - line_dash="dash", - line_color=COLORS["text_dim"], - opacity=0.6, - row=2, - col=1, - annotation_text=f"avg {avg_spread:+.4f}%", - annotation_font_color=COLORS["text_dim"], - ) - - fig.update_layout( - **LAYOUT_BASE, - title=dict( - text=( - f"BTC-BRL Spread | BTC: ${current_btc:,.0f}" - f" | Now: {curr_spread:+.4f}%" - f" | Range: [{min_spread:+.4f}%, {max_spread:+.4f}%]" - ), - font=dict(color=COLORS["text"], size=13), - x=0.5, - ), - width=1300, - height=700, - ) - fig.update_yaxes(title_text="USD", tickformat=",.0f", row=1, col=1) - fig.update_yaxes(title_text="%", tickformat=".4f", row=2, col=1) - for ann in fig.layout.annotations: - ann.font = dict(color=COLORS["text_dim"], size=11) - _style_axes(fig, 2) - return fig - - -def build_strategy_chart( - sim: pd.DataFrame, fee_bps: float, min_spread_bps: float, trade_size_usdt: float -) -> go.Figure: - """Chart 2: Strategy simulation results.""" - total_trades = int(sim["trade_count"].iloc[-1]) - final_pnl = sim["cum_pnl"].iloc[-1] - final_captured = sim["cum_captured"].iloc[-1] - final_fees = sim["cum_fees"].iloc[-1] - final_position = sim["position_usdt"].iloc[-1] - final_total = sim["total_pnl"].iloc[-1] - duration_min = (sim["ts"].iloc[-1] - sim["ts"].iloc[0]).total_seconds() / 60 - - fig = make_subplots( - rows=3, - cols=1, - row_heights=[0.4, 0.3, 0.3], - vertical_spacing=0.06, - subplot_titles=[ - f"Cumulative PnL — {total_trades} trades in {duration_min:.0f}min", - f"Net Position (max ${sim['position_usdt'].abs().max():,.0f})", - "Spread (bps) + Trade Signals", - ], - ) - - # Row 1: PnL breakdown - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["cum_captured"], - line=dict(color=COLORS["brl"], width=1.5), - name="Captured (gross)", - fill="tozeroy", - fillcolor="rgba(63,185,80,0.08)", - ), - row=1, - col=1, - ) - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["cum_fees"], - line=dict(color=COLORS["fees"], width=1, dash="dot"), - name="Fees paid", - ), - row=1, - col=1, - ) - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["cum_pnl"], - line=dict(color=COLORS["pnl"], width=2), - name="Net PnL (realized)", - ), - row=1, - col=1, - ) - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["total_pnl"], - line=dict(color=COLORS["spread"], width=1.5, dash="dash"), - name="Total PnL (realized + unrealized)", - ), - row=1, - col=1, - ) - - # Row 2: Position - pos_colors = [ - COLORS["long"] if v >= 0 else COLORS["short"] for v in sim["position_usdt"] - ] - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["position_usdt"], - line=dict(color=COLORS["pnl"], width=1.5), - fill="tozeroy", - fillcolor="rgba(88,166,255,0.08)", - name="Position (USDT)", - showlegend=True, - ), - row=2, - col=1, - ) - fig.add_hline(y=0, line_color=COLORS["grid"], row=2, col=1) - - # Row 3: Spread + signals - fig.add_trace( - go.Scatter( - x=sim["ts"], - y=sim["spread_bps"], - line=dict(color=COLORS["spread"], width=1), - name="Spread (bps)", - showlegend=True, - ), - row=3, - col=1, - ) - fig.add_hline( - y=min_spread_bps, - line_dash="dash", - line_color=COLORS["brl"], - opacity=0.5, - row=3, - col=1, - ) - fig.add_hline( - y=-min_spread_bps, - line_dash="dash", - line_color=COLORS["brl"], - opacity=0.5, - row=3, - col=1, - ) - fig.add_hline(y=0, line_color=COLORS["grid"], row=3, col=1) - - # Trade markers - sells_brl = sim[sim["action"] == "sell_brl"] - sells_usdt = sim[sim["action"] == "sell_usdt"] - if not sells_brl.empty: - fig.add_trace( - go.Scatter( - x=sells_brl["ts"], - y=sells_brl["spread_bps"], - mode="markers", - marker=dict(color=COLORS["short"], size=3, symbol="triangle-down"), - name="Sell BRL", - showlegend=True, - ), - row=3, - col=1, - ) - if not sells_usdt.empty: - fig.add_trace( - go.Scatter( - x=sells_usdt["ts"], - y=sells_usdt["spread_bps"], - mode="markers", - marker=dict(color=COLORS["long"], size=3, symbol="triangle-up"), - name="Sell USDT", - showlegend=True, - ), - row=3, - col=1, - ) - - fig.update_layout( - **LAYOUT_BASE, - title=dict( - text=( - f"Quoting Strategy Sim" - f" | ${trade_size_usdt:.0f}/trade" - f" | fee: {fee_bps:.0f}bps" - f" | min: {min_spread_bps:.0f}bps" - f" | PnL: ${final_pnl:+.2f}" - f" | Pos: ${final_position:+,.0f}" - ), - font=dict(color=COLORS["text"], size=12), - x=0.5, - ), - width=1300, - height=900, - ) - fig.update_yaxes(title_text="USDT", tickformat=",.2f", row=1, col=1) - fig.update_yaxes(title_text="USDT", tickformat=",.0f", row=2, col=1) - fig.update_yaxes(title_text="bps", row=3, col=1) - for ann in fig.layout.annotations: - ann.font = dict(color=COLORS["text_dim"], size=11) - _style_axes(fig, 3) - return fig - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: - chat_id = context._chat_id - client = await get_client(chat_id, context=context) - if not client: - return RoutineResult(text="No server available") - - # Fetch 1s candles in parallel - btc_usdt, btc_brl, usdt_brl = await asyncio.gather( - _fetch_candles( - client, config.connector_name, "BTC-USDT", "1s", config.lookback_seconds - ), - _fetch_candles( - client, config.connector_name, "BTC-BRL", "1s", config.lookback_seconds - ), - _fetch_candles( - client, config.connector_name, "USDT-BRL", "1s", config.lookback_seconds - ), - ) - - for name, df in [ - ("BTC-USDT", btc_usdt), - ("BTC-BRL", btc_brl), - ("USDT-BRL", usdt_brl), - ]: - if df.empty: - return RoutineResult(text=f"No 1s data for {name}") - - # Compute spread - merged = _merge_spread(btc_usdt, btc_brl, usdt_brl) - if merged.empty: - return RoutineResult(text="No overlapping data between pairs") - - # Run strategy simulation - sim = simulate_quoting( - merged, - trade_size_usdt=config.trade_size_usdt, - fee_bps=config.fee_bps, - min_spread_bps=config.min_spread_bps, - max_position_usdt=config.max_position_usdt, - ) - - # Build charts - spread_fig = build_spread_chart(merged) - strategy_fig = build_strategy_chart( - sim, config.fee_bps, config.min_spread_bps, config.trade_size_usdt - ) - - # Summary stats - duration_min = (merged["ts"].iloc[-1] - merged["ts"].iloc[0]).total_seconds() / 60 - avg_spread = merged["spread_pct"].mean() - std_spread = merged["spread_pct"].std() - curr_spread = merged["spread_pct"].iloc[-1] - total_trades = int(sim["trade_count"].iloc[-1]) - final_captured = sim["cum_captured"].iloc[-1] - final_fees = sim["cum_fees"].iloc[-1] - final_pnl = sim["cum_pnl"].iloc[-1] - final_position = sim["position_usdt"].iloc[-1] - final_total = sim["total_pnl"].iloc[-1] - - # Spread distribution - above_thresh = (merged["spread_bps"].abs() > config.min_spread_bps).mean() * 100 - time_positive = (merged["spread_pct"] > 0).mean() * 100 - - # Annualize PnL - hours = duration_min / 60 - if hours > 0: - pnl_per_hour = final_pnl / hours - projected_daily = pnl_per_hour * 24 - projected_monthly = projected_daily * 30 - else: - pnl_per_hour = projected_daily = projected_monthly = 0 - - text = "\n".join( - [ - "BTC-BRL Arbitrage Analysis", - f"Window: {duration_min:.0f} min ({len(merged):,} ticks)", - "", - "Spread Stats:", - f" Current: {curr_spread:+.4f}% ({curr_spread*100:+.2f} bps)", - f" Average: {avg_spread:+.4f}%", - f" Std Dev: {std_spread:.4f}%", - f" Time |spread| > {config.min_spread_bps:.0f}bps: {above_thresh:.1f}%", - f" Time BRL premium: {time_positive:.1f}%", - "", - f"Strategy Sim (${config.trade_size_usdt:.0f}/trade, {config.fee_bps:.0f}bps fee, >{config.min_spread_bps:.0f}bps):", - f" Trades: {total_trades}", - f" Gross captured: ${final_captured:.2f}", - f" Fees paid: ${final_fees:.2f}", - f" Net realized PnL: ${final_pnl:+.2f}", - f" Current position: ${final_position:+,.0f}", - f" Total PnL (incl. unrealized): ${final_total:+.2f}", - "", - "Projections (if conditions persist):", - f" Per hour: ${pnl_per_hour:+.2f}", - f" Per day: ${projected_daily:+.2f}", - f" Per month: ${projected_monthly:+.2f}", - ] - ) - - # Export primary chart - img_bytes = spread_fig.to_image(format="png", width=1300, height=700, scale=2) - chart = io.BytesIO(img_bytes) - - # Report - from condor.reports import ReportBuilder - - builder = ReportBuilder("BTC-BRL Arbitrage") - builder.source("routine", "btc_brl_arbitrage").tags( - ["btc", "brl", "arbitrage", "strategy"] - ) - builder.markdown(text) - builder.plotly(spread_fig) - builder.plotly(strategy_fig) - await builder.save() - - return RoutineResult(text=text, chart_image=chart) diff --git a/agents/brigado/routines/config_analyzer.py b/agents/brigado/routines/config_analyzer.py deleted file mode 100644 index ca811cd0c..000000000 --- a/agents/brigado/routines/config_analyzer.py +++ /dev/null @@ -1,772 +0,0 @@ -"""PMM Config Analyzer — Compare controller configs side by side, highlight differences.""" - -CATEGORY = "Bot Analysis" - -import asyncio -import logging -import re -from collections import defaultdict - -import plotly.graph_objects as go -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from config_manager import get_config_manager - -logger = logging.getLogger(__name__) - -_DARK_BG = "#1a1a2e" -_DARK_PAPER = "#16213e" -_GREEN = "#00d4aa" -_RED = "#ff6b6b" -_BLUE = "#58a6ff" -_YELLOW = "#ffdd57" -_COLORS = [ - "#58a6ff", - "#00d4aa", - "#ff6b6b", - "#ffdd57", - "#c084fc", - "#fb923c", - "#38bdf8", - "#f472b6", -] - -# Parameters to always skip in comparison (noise) -SKIP_PARAMS = {"id", "controller_type", "controller_name", "manual_kill_switch"} - -# Parameters to highlight when different (key tuning params) -KEY_PARAMS = { - "buy_spreads", - "sell_spreads", - "buy_amounts_pct", - "sell_amounts_pct", - "total_amount_quote", - "stop_loss", - "take_profit", - "time_limit", - "executor_refresh_time", - "cooldown_time", - "leverage", - "dca_spreads", - "dca_amounts", - "trailing_stop", - "executor_activation_bounds", - "top_executor_refresh_time", - "position_rebalance_threshold_pct", -} - - -class Config(BaseModel): - """Analyze and compare PMM controller configs — saved vs running bots.""" - - server_name: str = Field( - default="brigado_2", - description="Server to connect to", - ) - controller_type: str = Field( - default="", - description="Filter by controller type (e.g. 'market_making'). Empty = all types.", - ) - controller_filter: str = Field( - default="pmm_mister", - description="Filter by config name or controller_name containing this string. Empty = all.", - ) - pair_filter: str = Field( - default="", - description="Filter by trading pair (e.g. 'BRL', 'BTC-BRL'). Empty = all.", - ) - running_only: bool = Field( - default=False, - description="Show only configs that are currently running in active bots.", - ) - show_common: bool = Field( - default=False, - description="Include parameters that are the same across all configs (verbose mode).", - ) - - -def _short_name(config_name: str) -> str: - """Shorten config name for display.""" - name = config_name - name = re.sub(r"^mm_", "", name) - name = re.sub(r"pmm_?mister_?", "pm_", name) - name = re.sub(r"^(binance|okx|bybit|kucoin)(_perpetual)?_", "", name) - name = re.sub(r"config[_]?(\d+)", r"c\1", name) - while "__" in name: - name = name.replace("__", "_") - return name.strip("_") or config_name[:25] - - -def _fmt_value(v) -> str: - """Format a config value for display.""" - if v is None: - return "—" - if isinstance(v, bool): - return "✓" if v else "✗" - if isinstance(v, float): - if v < 0.001: - return f"{v:.5f}" - if v < 0.01: - return f"{v:.4f}" - return f"{v:.3f}" if v < 10 else f"{v:.1f}" - if isinstance(v, list): - return ", ".join(_fmt_value(x) for x in v) - if isinstance(v, dict): - parts = [f"{k}={_fmt_value(vv)}" for k, vv in v.items()] - return " | ".join(parts) - return str(v) - - -def _normalize_value(v): - """Normalize values for comparison.""" - if v is None: - return None - if isinstance(v, str): - try: - return float(v) - except ValueError: - return v.strip().lower() - if isinstance(v, list): - return [_normalize_value(x) for x in v] - if isinstance(v, dict): - return {k: _normalize_value(vv) for k, vv in v.items()} - return v - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> str: - # Connect to server - client = await get_config_manager().get_client(config.server_name) - if not client: - return f"Could not connect to server '{config.server_name}'" - - # Fetch all data in parallel - all_configs_list, bots_data = await asyncio.gather( - client.controllers.list_controller_configs(), - _get_bots_data(client), - ) - - # --- Saved configs --- - saved_configs = {} - filt = config.controller_filter.lower() if config.controller_filter else "" - for cfg_meta in all_configs_list or []: - cname = cfg_meta.get("id") or cfg_meta.get("config_name", "") - ctype = cfg_meta.get("controller_type", "") - ctrl_name = cfg_meta.get("controller_name", "") - - if ( - config.controller_type - and config.controller_type.lower() not in ctype.lower() - ): - continue - if filt and filt not in cname.lower() and filt not in ctrl_name.lower(): - continue - - try: - cfg_data = await client.controllers.get_controller_config(cname) - # Also check controller_name inside the full config data - if filt and filt not in cname.lower(): - inner_ctrl = cfg_data.get("controller_name", "") - if filt not in inner_ctrl.lower(): - continue - if config.pair_filter: - pair = cfg_data.get("trading_pair", "") - if config.pair_filter.upper() not in pair.upper(): - continue - saved_configs[cname] = cfg_data - except Exception as e: - logger.warning(f"Failed to fetch config {cname}: {e}") - - # --- Running bot configs --- - running_config_ids = set() - running_configs = {} - for bot_name in bots_data or {}: - try: - bot_cfgs = await client.controllers.get_bot_controller_configs(bot_name) - if isinstance(bot_cfgs, list): - for cfg in bot_cfgs: - cid = cfg.get("id") or cfg.get("controller_id", "") - ctype = cfg.get("controller_type", "") - if ( - config.controller_type - and config.controller_type.lower() not in ctype.lower() - ): - continue - ctrl_name_inner = cfg.get("controller_name", "") - if ( - filt - and filt not in cid.lower() - and filt not in ctrl_name_inner.lower() - ): - continue - if config.pair_filter: - pair = cfg.get("trading_pair", "") - if config.pair_filter.upper() not in pair.upper(): - continue - running_configs[cid] = cfg - running_config_ids.add(cid) - except Exception as e: - logger.warning(f"Failed to fetch bot configs for {bot_name}: {e}") - - # Merge all configs with status prefix - all_configs = {} - for name, cfg in saved_configs.items(): - prefix = "🟢" if name in running_config_ids else "🔵" - all_configs[f"{prefix} {name}"] = cfg - for cid, cfg in running_configs.items(): - if cid not in saved_configs: - all_configs[f"🟢 {cid}"] = cfg - - # Filter to running only if requested - if config.running_only: - all_configs = { - k: v for k, v in all_configs.items() if k.startswith("\U0001f7e2") - } - - if not all_configs: - filters = [] - if config.controller_type: - filters.append(f"type={config.controller_type}") - if config.controller_filter: - filters.append(f"filter={config.controller_filter}") - if config.pair_filter: - filters.append(f"pair={config.pair_filter}") - return f"No configs found matching: {', '.join(filters) if filters else 'any'}" - - # --- Build comparison --- - all_params = set() - for cfg in all_configs.values(): - all_params.update(cfg.keys()) - all_params -= SKIP_PARAMS - - # Sort: key params first, then alphabetical - sorted_params = sorted(all_params, key=lambda p: (0 if p in KEY_PARAMS else 1, p)) - - # Find which params differ - diff_params = set() - for param in sorted_params: - values = [_normalize_value(cfg.get(param)) for cfg in all_configs.values()] - non_none = [v for v in values if v is not None] - if len(set(str(v) for v in non_none)) > 1: - diff_params.add(param) - - # Group configs by trading pair - pair_groups = defaultdict(dict) - for label, cfg in all_configs.items(): - pair = cfg.get("trading_pair", "unknown") - pair_groups[pair][label] = cfg - - num_running = sum(1 for l in all_configs if l.startswith("🟢")) - - # --- Build HTML report --- - from condor.reports import ReportBuilder - - builder = ReportBuilder("PMM Config Analyzer") - builder.source("routine", "config_analyzer").tags(["configs", "pmm", "analysis"]) - builder.manual_order() - - # KPIs - builder.kpi("Total Configs", str(len(all_configs))) - builder.kpi("Running", str(num_running), trend="up" if num_running else "neutral") - builder.kpi("Saved Only", str(len(all_configs) - num_running)) - builder.kpi( - "Diffs Found", - str(len(diff_params)), - delta=f"of {len(sorted_params)} params", - ) - builder.kpi("Trading Pairs", str(len(pair_groups))) - - # --- Per-pair comparison tables --- - for pair, pair_cfgs in sorted(pair_groups.items(), key=lambda x: -len(x[1])): - n_running = sum(1 for l in pair_cfgs if l.startswith("🟢")) - - labels = list(pair_cfgs.keys()) - shorts = [_short_name(l.replace("🟢 ", "").replace("🔵 ", "")) for l in labels] - - # Find pair-level diffs - pair_diff_params = set() - for param in sorted_params: - values = [_normalize_value(pair_cfgs[l].get(param)) for l in labels] - non_none = [v for v in values if v is not None] - if len(non_none) > 0 and len(set(str(v) for v in non_none)) > 1: - pair_diff_params.add(param) - - heading_md = f"## {pair} ({len(pair_cfgs)} configs, {n_running} running)" - - if len(labels) == 1: - # Single config — show full params - label = labels[0] - cfg = pair_cfgs[label] - rows = [] - for param in sorted_params: - val = cfg.get(param) - if val is not None: - is_key = "⚡ " if param in KEY_PARAMS else "" - rows.append( - { - "Parameter": f"{is_key}{param}", - "Value": _fmt_value(val), - } - ) - builder.markdown(f"{heading_md}\n\n**{label}** — single config") - if rows: - builder.table(rows) - else: - # Multiple configs — comparison view - rows = [] - for param in sorted_params: - values = [pair_cfgs[l].get(param) for l in labels] - non_none = [v for v in values if v is not None] - if not non_none: - continue - - is_diff = param in pair_diff_params - if not config.show_common and not is_diff: - continue - - prefix = ( - "🔴 " - if is_diff and param in KEY_PARAMS - else "🟡 " if is_diff else "" - ) - row = {"Parameter": f"{prefix}{param}"} - for short, val in zip(shorts, values): - row[short] = _fmt_value(val) - rows.append(row) - - note = ( - f"Showing {len(rows)} differing params. 🔴 = key, 🟡 = other." - if not config.show_common - else "" - ) - if not rows and not config.show_common: - builder.markdown( - f"{heading_md}\n\nAll parameters identical across these configs." - ) - else: - builder.markdown(f"{heading_md}\n\n{note}" if note else heading_md) - if rows: - builder.table(rows) - - # --- Cross-pair key differences summary --- - if len(pair_groups) > 1 and diff_params: - cross_sections = [] - for param in sorted_params: - if param not in diff_params or param not in KEY_PARAMS: - continue - value_groups = defaultdict(list) - for label, cfg in all_configs.items(): - val = _fmt_value(cfg.get(param)) - short = _short_name(label.replace("🟢 ", "").replace("🔵 ", "")) - value_groups[val].append(short) - if len(value_groups) <= 1: - continue - rows = [] - for val, names in sorted(value_groups.items(), key=lambda x: -len(x[1])): - rows.append( - { - "Value": val, - "Count": str(len(names)), - "Configs": ", ".join(names[:8]) - + (f" (+{len(names)-8})" if len(names) > 8 else ""), - } - ) - cross_sections.append((param, rows)) - - if cross_sections: - builder.markdown( - "## Cross-Pair Key Differences\n\nKey tuning parameters grouped by distinct values across all configs." - ) - for param, rows in cross_sections: - builder.markdown(f"**`{param}`** — {len(rows)} distinct values:") - builder.table(rows) - - # --- Charts --- - if len(all_configs) > 1: - fig_pc = _build_parallel_coords(all_configs, running_config_ids) - if fig_pc: - builder.plotly(fig_pc) - - fig = _build_spread_chart(all_configs) - if fig: - builder.plotly(fig) - - fig2 = _build_capital_chart(all_configs) - if fig2: - builder.plotly(fig2) - - await builder.save() - - # --- Concise plain text summary --- - lines = [f"**PMM Config Analyzer** — {config.server_name}", ""] - lines.append( - f"Found **{len(all_configs)}** configs ({num_running} running, {len(all_configs) - num_running} saved-only)" - ) - lines.append(f"**{len(diff_params)}** parameters differ across configs") - lines.append( - f"**{len(pair_groups)}** pairs: {', '.join(sorted(pair_groups.keys()))}" - ) - lines.append("") - - for pair, pair_cfgs in sorted(pair_groups.items(), key=lambda x: -len(x[1])): - n_r = sum(1 for l in pair_cfgs if l.startswith("🟢")) - lines.append(f"- **{pair}**: {len(pair_cfgs)} configs ({n_r} running)") - - lines.append("") - lines.append("Full comparison report available in the dashboard.") - - return "\n".join(lines) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -async def _get_bots_data(client) -> dict: - try: - data = await client.bot_orchestration.get_active_bots_status() - if isinstance(data, dict) and "data" in data: - return data["data"] - except Exception: - pass - return {} - - -def _build_parallel_coords(all_configs: dict, running_ids: set) -> go.Figure | None: - """Parallel coordinates plot (HiPlot-style) for numeric config parameters.""" - # Extract numeric axes from key params - NUMERIC_AXES = [ - ("total_amount_quote", "Capital", None), - ( - "buy_spreads", - "Buy Spread L1 %", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "sell_spreads", - "Sell Spread L1 %", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "buy_spreads", - "Buy Spread L2 %", - lambda v: ( - (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None - ), - ), - ( - "sell_spreads", - "Sell Spread L2 %", - lambda v: ( - (v[1] if isinstance(v, list) and len(v) > 1 else None) if v else None - ), - ), - ("executor_refresh_time", "Refresh Time", None), - ("cooldown_time", "Cooldown", None), - ("leverage", "Leverage", None), - ("stop_loss", "Stop Loss", None), - ("take_profit", "Take Profit", None), - ("time_limit", "Time Limit", None), - ("top_executor_refresh_time", "Top Refresh", None), - ( - "dca_spreads", - "DCA Spread L1", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "dca_amounts", - "DCA Amt L1", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ( - "trailing_stop", - "Trail Stop", - lambda v: ( - (v.get("activation_price_delta") if isinstance(v, dict) else v) - if v - else None - ), - ), - ( - "executor_activation_bounds", - "Activation Bounds", - lambda v: (v[0] if isinstance(v, list) and v else v) if v else None, - ), - ] - - # Collect data rows - rows = [] - labels = [] - for label, cfg in all_configs.items(): - clean_label = label.replace("\U0001f7e2 ", "").replace("\U0001f535 ", "") - is_running = clean_label in running_ids or label.startswith("\U0001f7e2") - pair = cfg.get("trading_pair", "unknown") - row = {"_running": 1 if is_running else 0, "_pair": pair} - for param, axis_name, extractor in NUMERIC_AXES: - raw = cfg.get(param) - if extractor: - val = extractor(raw) - else: - val = raw - try: - row[axis_name] = float(val) if val is not None else None - except (TypeError, ValueError): - row[axis_name] = None - rows.append(row) - labels.append(_short_name(clean_label)) - - if len(rows) < 2: - return None - - # Filter axes that have at least 2 distinct non-None values - valid_axes = [] - for _, axis_name, _ in NUMERIC_AXES: - vals = [r[axis_name] for r in rows if r[axis_name] is not None] - if len(vals) >= 2 and len(set(vals)) > 1: - valid_axes.append(axis_name) - - if len(valid_axes) < 2: - return None - - # Build color by pair - unique_pairs = sorted(set(r["_pair"] for r in rows)) - pair_to_idx = {p: i for i, p in enumerate(unique_pairs)} - color_values = [pair_to_idx[r["_pair"]] for r in rows] - - # Build colorscale from our palette - n_pairs = len(unique_pairs) - if n_pairs == 1: - colorscale = [[0, _BLUE], [1, _BLUE]] - else: - colorscale = [] - for i in range(n_pairs): - pos = i / (n_pairs - 1) - colorscale.append([pos, _COLORS[i % len(_COLORS)]]) - - # Fill None values with axis min (so lines don't break) - for axis_name in valid_axes: - vals = [r[axis_name] for r in rows if r[axis_name] is not None] - fill = min(vals) if vals else 0 - for r in rows: - if r[axis_name] is None: - r[axis_name] = fill - - # Build dimensions - dimensions = [] - for axis_name in valid_axes: - vals = [r[axis_name] for r in rows] - # Multiply spread values by 100 for readability - if "%" in axis_name: - vals = [v * 100 for v in vals] - dimensions.append( - dict( - label=axis_name, - values=vals, - range=( - [min(vals), max(vals)] - if min(vals) != max(vals) - else [min(vals) - 1, max(vals) + 1] - ), - ) - ) - - fig = go.Figure( - data=go.Parcoords( - line=dict( - color=color_values, - colorscale=colorscale, - showscale=False, - ), - dimensions=dimensions, - customdata=labels, - ) - ) - - # Add pair legend via invisible scatter traces - for i, pair in enumerate(unique_pairs): - fig.add_trace( - go.Scatter( - x=[None], - y=[None], - mode="markers", - marker=dict(size=10, color=_COLORS[i % len(_COLORS)]), - name=pair, - showlegend=True, - ) - ) - - fig.update_layout( - title=dict( - text=f"Config Parameter Space ({len(rows)} configs, {len(valid_axes)} axes)", - font=dict(color="#e6edf3", size=14), - x=0.5, - ), - paper_bgcolor=_DARK_PAPER, - plot_bgcolor=_DARK_BG, - font=dict(color="#ccc", size=11), - margin=dict(l=80, r=80, t=80, b=40), - width=1400, - height=600, - legend=dict( - orientation="h", y=-0.05, x=0.5, xanchor="center", font=dict(size=12) - ), - ) - return fig - - -def _build_spread_chart(all_configs: dict) -> go.Figure | None: - """Grouped bar chart comparing buy/sell spreads across configs.""" - - def _to_list(v) -> list: - if v is None: - return [] - if isinstance(v, (int, float)): - return [float(v)] - if isinstance(v, list): - return [float(x) for x in v] - return [] - - configs_with_spreads = [] - for label, cfg in all_configs.items(): - buy = _to_list(cfg.get("buy_spreads")) - sell = _to_list(cfg.get("sell_spreads")) - if buy or sell: - short = _short_name(label.replace("🟢 ", "").replace("🔵 ", "")) - pair = cfg.get("trading_pair", "") - configs_with_spreads.append( - { - "name": short, - "pair": pair, - "buy_l1": buy[0] * 100 if len(buy) >= 1 else 0, - "sell_l1": sell[0] * 100 if len(sell) >= 1 else 0, - "buy_l2": buy[1] * 100 if len(buy) >= 2 else 0, - "sell_l2": sell[1] * 100 if len(sell) >= 2 else 0, - } - ) - - if not configs_with_spreads: - return None - - configs_with_spreads.sort(key=lambda c: (c["pair"], c["name"])) - names = [c["name"] for c in configs_with_spreads] - - fig = go.Figure() - fig.add_trace( - go.Bar( - x=names, - y=[c["buy_l1"] for c in configs_with_spreads], - name="Buy L1", - marker_color=_GREEN, - opacity=0.9, - ) - ) - fig.add_trace( - go.Bar( - x=names, - y=[c["sell_l1"] for c in configs_with_spreads], - name="Sell L1", - marker_color=_RED, - opacity=0.9, - ) - ) - - has_l2 = any(c["buy_l2"] > 0 or c["sell_l2"] > 0 for c in configs_with_spreads) - if has_l2: - fig.add_trace( - go.Bar( - x=names, - y=[c["buy_l2"] for c in configs_with_spreads], - name="Buy L2", - marker_color=_GREEN, - opacity=0.5, - ) - ) - fig.add_trace( - go.Bar( - x=names, - y=[c["sell_l2"] for c in configs_with_spreads], - name="Sell L2", - marker_color=_RED, - opacity=0.5, - ) - ) - - fig.update_layout( - title=dict( - text="Spread Levels by Config (%)", - font=dict(color="#e6edf3", size=14), - x=0.5, - ), - paper_bgcolor=_DARK_PAPER, - plot_bgcolor=_DARK_BG, - font=dict(color="#ccc", size=10), - barmode="group", - yaxis=dict(gridcolor="#2a2a4a", title="Spread %"), - xaxis=dict(gridcolor="#2a2a4a", tickangle=-45), - legend=dict(orientation="h", y=1.1, x=0.5, xanchor="center"), - margin=dict(l=60, r=40, t=60, b=120), - width=1200, - height=500, - ) - return fig - - -def _build_capital_chart(all_configs: dict) -> go.Figure | None: - """Horizontal bar chart: capital allocation per config, colored by pair.""" - data = [] - for label, cfg in all_configs.items(): - amt = float(cfg.get("total_amount_quote", 0) or 0) - if amt <= 0: - continue - short = _short_name(label.replace("🟢 ", "").replace("🔵 ", "")) - pair = cfg.get("trading_pair", "") - is_running = label.startswith("🟢") - data.append({"name": short, "pair": pair, "amount": amt, "running": is_running}) - - if not data: - return None - - data.sort(key=lambda d: (d["pair"], d["amount"])) - - unique_pairs = list(dict.fromkeys(d["pair"] for d in data)) - pair_colors = {p: _COLORS[i % len(_COLORS)] for i, p in enumerate(unique_pairs)} - - fig = go.Figure() - for pair in unique_pairs: - pair_data = [d for d in data if d["pair"] == pair] - fig.add_trace( - go.Bar( - y=[d["name"] for d in pair_data], - x=[d["amount"] for d in pair_data], - orientation="h", - name=pair, - marker_color=pair_colors[pair], - opacity=0.85, - text=[ - f"{'🟢' if d['running'] else ''} {d['amount']:,.0f}" - for d in pair_data - ], - textposition="auto", - ) - ) - - height = max(400, len(data) * 22 + 100) - fig.update_layout( - title=dict( - text="Capital Allocation by Config (quote currency)", - font=dict(color="#e6edf3", size=14), - x=0.5, - ), - paper_bgcolor=_DARK_PAPER, - plot_bgcolor=_DARK_BG, - font=dict(color="#ccc", size=10), - barmode="stack", - xaxis=dict(gridcolor="#2a2a4a", title="Amount (quote)"), - yaxis=dict(gridcolor="#2a2a4a"), - legend=dict(orientation="h", y=1.05, x=0.5, xanchor="center"), - margin=dict(l=150, r=40, t=60, b=50), - width=1200, - height=height, - ) - return fig diff --git a/agents/brigado/routines/spot_markets.py b/agents/brigado/routines/spot_markets.py deleted file mode 100644 index a3795b995..000000000 --- a/agents/brigado/routines/spot_markets.py +++ /dev/null @@ -1,458 +0,0 @@ -"""Spot market discovery — top markets by volume across quote currencies with rebate focus.""" - -CATEGORY = "Market Data" - -import asyncio -import logging -from typing import Any - -import aiohttp -import pandas as pd -import plotly.graph_objects as go -from plotly.subplots import make_subplots -from pydantic import BaseModel, Field -from telegram.ext import ContextTypes - -from config_manager import get_client -from routines.base import RoutineResult - -logger = logging.getLogger(__name__) - -BINANCE_SPOT_TICKER = "https://api.binance.com/api/v3/ticker/24hr" - -# Quote currencies we care about -REBATE_QUOTES = { - "BRL", - "ARS", - "MXN", - "EUR", - "PLN", - "JPY", - "TRY", - "RON", - "UAH", - "ZAR", - "NGN", - "COP", -} -STABLE_QUOTES = {"USDT", "USDC", "BUSD", "FDUSD", "TUSD"} - -# Stablecoins / wrapped tokens to exclude as base -EXCLUDED_BASES = {"USDT", "USDC", "BUSD", "TUSD", "FDUSD", "DAI", "BFUSD"} - -COLORS = { - "bg_paper": "#0d1117", - "bg_plot": "#161b22", - "grid": "#21262d", - "text": "#c9d1d9", - "text_dim": "#8b949e", - "green": "#3fb950", - "red": "#f85149", - "blue": "#58a6ff", - "yellow": "#f0b90b", - "purple": "#bc8cff", - "orange": "#f78166", -} - -QUOTE_COLORS = { - "BRL": "#3fb950", - "USDT": "#e6e6e6", - "FDUSD": "#58a6ff", - "USDC": "#2775ca", - "ARS": "#75b5f0", - "MXN": "#f0b90b", - "EUR": "#f78166", - "PLN": "#bc8cff", - "JPY": "#ff7b72", - "TRY": "#d2a8ff", - "RON": "#7ee787", - "UAH": "#79c0ff", - "ZAR": "#ffa657", - "NGN": "#56d364", - "COP": "#e3b341", -} - - -class Config(BaseModel): - """Spot market discovery — top pairs by volume, FIAT rebate markets, and USDT leaders.""" - - top_n: int = Field(default=50, description="Top N markets per section") - min_volume_usd: float = Field( - default=100_000, description="Min 24h volume (USD) for FIAT pairs" - ) - min_volume_usdt: float = Field( - default=5_000_000, description="Min 24h volume (USD) for USDT pairs" - ) - focus_quote: str = Field( - default="BRL", description="Primary FIAT quote to highlight" - ) - - -def _parse_quote(symbol: str, known_quotes: set[str]) -> tuple[str, str] | None: - """Parse a Binance symbol into (base, quote) using known quote list.""" - for q in sorted(known_quotes, key=len, reverse=True): - if symbol.endswith(q): - base = symbol[: -len(q)] - if base: - return base, q - return None - - -async def fetch_spot_tickers() -> list[dict]: - """Fetch all 24h tickers from Binance spot.""" - async with aiohttp.ClientSession() as session: - async with session.get(BINANCE_SPOT_TICKER) as resp: - resp.raise_for_status() - return await resp.json() - - -def categorize_tickers(tickers: list[dict]) -> dict[str, list[dict]]: - """Parse and categorize tickers by quote currency.""" - all_quotes = REBATE_QUOTES | STABLE_QUOTES | {"BTC", "ETH", "BNB"} - by_quote: dict[str, list[dict]] = {} - - for t in tickers: - symbol = t.get("symbol", "") - parsed = _parse_quote(symbol, all_quotes) - if not parsed: - continue - base, quote = parsed - if base in EXCLUDED_BASES: - continue - - quote_vol = float(t.get("quoteVolume", 0)) - if quote_vol <= 0: - continue - - entry = { - "symbol": symbol, - "base": base, - "quote": quote, - "trading_pair": f"{base}-{quote}", - "price": float(t.get("lastPrice", 0)), - "price_change_pct": float(t.get("priceChangePercent", 0)), - "volume_24h": float(t.get("volume", 0)), - "quote_volume_24h": quote_vol, - "trades_24h": int(t.get("count", 0)), - } - by_quote.setdefault(quote, []).append(entry) - - # Sort each group by quote volume - for q in by_quote: - by_quote[q].sort(key=lambda x: x["quote_volume_24h"], reverse=True) - - return by_quote - - -async def _fetch_usdt_rates(tickers: list[dict]) -> dict[str, float]: - """Extract USDT rates for fiat currencies from ticker data.""" - rates = {"USDT": 1.0, "USDC": 1.0, "FDUSD": 1.0, "BUSD": 1.0, "TUSD": 1.0} - # BTC-USDT price for cross-conversion - btc_usdt = 0.0 - for t in tickers: - sym = t.get("symbol", "") - price = float(t.get("lastPrice", 0)) - if sym == "BTCUSDT": - btc_usdt = price - # Direct USDT/FIAT pairs (e.g., USDTBRL → 1 USDT = X BRL) - if sym.startswith("USDT") and price > 0: - fiat = sym[4:] - if fiat in REBATE_QUOTES: - rates[fiat] = 1.0 / price # BRL per USDT → USDT per BRL - # Also check FIAT/USDT pairs (unlikely on Binance but just in case) - for t in tickers: - sym = t.get("symbol", "") - price = float(t.get("lastPrice", 0)) - if sym.endswith("USDT") and price > 0: - base = sym[:-4] - if base in REBATE_QUOTES and base not in rates: - rates[base] = price - return rates - - -def format_volume(v: float) -> str: - if v >= 1_000_000_000: - return f"${v / 1_000_000_000:.1f}B" - if v >= 1_000_000: - return f"${v / 1_000_000:.1f}M" - if v >= 1_000: - return f"${v / 1_000:.0f}K" - return f"${v:.0f}" - - -def _build_report( - by_quote: dict[str, list[dict]], - usdt_rates: dict[str, float], - config: Config, -) -> tuple[str, go.Figure, list[dict]]: - """Build text summary, chart, and table data.""" - - # --- Section 1: Focus FIAT quote (BRL by default) --- - focus = config.focus_quote.upper() - focus_pairs = by_quote.get(focus, []) - focus_rate = usdt_rates.get(focus, 0) - - # --- Section 2: All FIAT markets summary --- - fiat_summary = [] - for q in sorted(REBATE_QUOTES): - pairs = by_quote.get(q, []) - if not pairs: - continue - rate = usdt_rates.get(q, 0) - total_vol_usdt = ( - sum(p["quote_volume_24h"] * rate for p in pairs) if rate > 0 else 0 - ) - fiat_summary.append( - { - "quote": q, - "pair_count": len(pairs), - "total_volume_usdt": total_vol_usdt, - "top_pair": pairs[0]["trading_pair"] if pairs else "-", - "top_volume": ( - pairs[0]["quote_volume_24h"] * rate if pairs and rate > 0 else 0 - ), - } - ) - fiat_summary.sort(key=lambda x: x["total_volume_usdt"], reverse=True) - - # --- Section 3: Top USDT pairs --- - usdt_pairs = [ - p - for p in by_quote.get("USDT", []) - if p["quote_volume_24h"] >= config.min_volume_usdt - ] - usdt_top = usdt_pairs[: config.top_n] - - # --- Build text --- - lines = [] - - # FIAT overview - lines.append("FIAT MARKETS OVERVIEW (Rebate Eligible)") - lines.append("-" * 45) - total_fiat_vol = sum(f["total_volume_usdt"] for f in fiat_summary) - lines.append(f"Total FIAT volume: {format_volume(total_fiat_vol)} USDT equiv.") - lines.append("") - for f in fiat_summary: - lines.append( - f" {f['quote']:>4s}: {f['pair_count']:3d} pairs | " - f"Vol: {format_volume(f['total_volume_usdt']):>8s} | " - f"Top: {f['top_pair']}" - ) - - # Focus FIAT detail - if focus_pairs and focus_rate > 0: - filtered = [ - p - for p in focus_pairs - if p["quote_volume_24h"] * focus_rate >= config.min_volume_usd - ] - lines.append("") - lines.append(f"TOP {focus}-QUOTED MARKETS (Rebates)") - lines.append("-" * 45) - for i, p in enumerate(filtered[: config.top_n], 1): - vol_usdt = p["quote_volume_24h"] * focus_rate - chg = p["price_change_pct"] - chg_s = f"+{chg:.1f}%" if chg >= 0 else f"{chg:.1f}%" - lines.append( - f" {i:2d}. {p['trading_pair']:<12s} " - f"Vol: {format_volume(vol_usdt):>8s} " - f"24h: {chg_s:>7s} " - f"Trades: {p['trades_24h']:,}" - ) - - # Top USDT - lines.append("") - lines.append(f"TOP {config.top_n} USDT-QUOTED MARKETS") - lines.append("-" * 45) - for i, p in enumerate(usdt_top, 1): - chg = p["price_change_pct"] - chg_s = f"+{chg:.1f}%" if chg >= 0 else f"{chg:.1f}%" - lines.append( - f" {i:2d}. {p['trading_pair']:<12s} " - f"Vol: {format_volume(p['quote_volume_24h']):>8s} " - f"24h: {chg_s:>7s} " - f"Trades: {p['trades_24h']:,}" - ) - - text = "\n".join(lines) - - # --- Chart: two panels --- - fig = make_subplots( - rows=2, - cols=1, - subplot_titles=[ - f"FIAT Market Volume (USDT equiv.)", - f"Top 30 USDT Markets — Volume vs 24h Change", - ], - row_heights=[0.35, 0.65], - vertical_spacing=0.12, - ) - - # Panel 1: FIAT volume bar chart - if fiat_summary: - quotes = [f["quote"] for f in fiat_summary] - volumes = [f["total_volume_usdt"] for f in fiat_summary] - bar_colors = [QUOTE_COLORS.get(q, "#8b949e") for q in quotes] - fig.add_trace( - go.Bar( - x=quotes, - y=volumes, - marker_color=bar_colors, - text=[format_volume(v) for v in volumes], - textposition="outside", - textfont=dict(size=9, color=COLORS["text"]), - hovertemplate="%{x}: %{y:,.0f} USDT", - showlegend=False, - ), - row=1, - col=1, - ) - - # Panel 2: Scatter — volume vs price change for top USDT - scatter_data = usdt_top[:30] - if scatter_data: - volumes_s = [p["quote_volume_24h"] for p in scatter_data] - changes = [p["price_change_pct"] for p in scatter_data] - labels = [p["base"] for p in scatter_data] - scatter_colors = [COLORS["green"] if c >= 0 else COLORS["red"] for c in changes] - - fig.add_trace( - go.Scatter( - x=changes, - y=volumes_s, - mode="markers+text", - marker=dict( - color=scatter_colors, - size=[max(8, min(25, v / max(volumes_s) * 25)) for v in volumes_s], - line=dict(width=1, color=COLORS["bg_paper"]), - ), - text=labels, - textposition="top center", - textfont=dict(size=8, color=COLORS["text_dim"]), - hovertemplate="%{text}
Vol: %{y:,.0f}
Chg: %{x:.1f}%", - showlegend=False, - ), - row=2, - col=1, - ) - - fig.update_layout( - paper_bgcolor=COLORS["bg_paper"], - plot_bgcolor=COLORS["bg_plot"], - font=dict(color=COLORS["text_dim"], size=10), - margin=dict(l=60, r=30, t=60, b=40), - height=800, - ) - for i in range(1, 3): - fig.update_xaxes(gridcolor=COLORS["grid"], row=i, col=1) - fig.update_yaxes(gridcolor=COLORS["grid"], row=i, col=1) - fig.update_yaxes(type="log", title="Volume (USDT)", row=1, col=1) - fig.update_yaxes(type="log", title="Volume 24h (USDT)", row=2, col=1) - fig.update_xaxes(title="24h Change (%)", row=2, col=1) - - # Annotation styling for subplot titles - for ann in fig.layout.annotations: - ann.font = dict(size=12, color=COLORS["text"]) - - # --- Table data for report --- - table_rows = [] - - # FIAT summary table - for f in fiat_summary: - table_rows.append( - { - "Section": "FIAT Overview", - "Quote": f["quote"], - "Pairs": f["pair_count"], - "Total Volume": format_volume(f["total_volume_usdt"]), - "Top Pair": f["top_pair"], - } - ) - - # Focus pairs table - if focus_pairs and focus_rate > 0: - for p in focus_pairs[: config.top_n]: - vol_usdt = p["quote_volume_24h"] * focus_rate - if vol_usdt < config.min_volume_usd: - continue - table_rows.append( - { - "Section": f"{focus} Markets", - "Pair": p["trading_pair"], - "Volume (USDT)": format_volume(vol_usdt), - "24h Change": f"{p['price_change_pct']:+.1f}%", - "Trades": f"{p['trades_24h']:,}", - } - ) - - # USDT top pairs - for p in usdt_top[: config.top_n]: - table_rows.append( - { - "Section": "USDT Markets", - "Pair": p["trading_pair"], - "Volume (USDT)": format_volume(p["quote_volume_24h"]), - "24h Change": f"{p['price_change_pct']:+.1f}%", - "Trades": f"{p['trades_24h']:,}", - } - ) - - return text, fig, table_rows - - -async def run(config: Config, context: ContextTypes.DEFAULT_TYPE) -> RoutineResult: - """Discover spot markets across quote currencies.""" - # Fetch all tickers - try: - tickers = await fetch_spot_tickers() - except Exception as e: - return RoutineResult(text=f"Failed to fetch Binance spot tickers: {e}") - - if not tickers: - return RoutineResult(text="No ticker data returned from Binance.") - - # Categorize and get rates - by_quote = categorize_tickers(tickers) - usdt_rates = await _fetch_usdt_rates(tickers) - - # Build output - text, fig, table_rows = _build_report(by_quote, usdt_rates, config) - - # Generate report - from condor.reports import ReportBuilder - - # Split tables by section - fiat_overview = [r for r in table_rows if r.get("Section") == "FIAT Overview"] - focus_rows = [ - r - for r in table_rows - if r.get("Section") == f"{config.focus_quote.upper()} Markets" - ] - usdt_rows = [r for r in table_rows if r.get("Section") == "USDT Markets"] - - # Clean section column - for rows in [fiat_overview, focus_rows, usdt_rows]: - for r in rows: - r.pop("Section", None) - - builder = ReportBuilder("Spot Market Discovery") - builder.source("routine", "spot_markets").tags( - ["spot", "markets", "discovery", "fiat", "rebates"] - ) - builder.plotly(fig) - - builder.markdown("### FIAT Markets Overview (Rebate Eligible)") - if fiat_overview: - builder.table(fiat_overview) - - if focus_rows: - builder.markdown(f"### Top {config.focus_quote.upper()}-Quoted Markets") - builder.table(focus_rows) - - builder.markdown("### Top USDT-Quoted Markets") - if usdt_rows: - builder.table(usdt_rows) - - await builder.save() - - return RoutineResult(text=text) diff --git a/agents/brigado/routines/toxic_flow_detector.py b/agents/brigado/routines/toxic_flow_detector.py deleted file mode 100644 index e69de29bb..000000000 diff --git a/agents/brigado/strategies/brl_mm/strategy.md b/agents/brigado/strategies/brl_mm/strategy.md deleted file mode 100644 index ca1ffb6e1..000000000 --- a/agents/brigado/strategies/brl_mm/strategy.md +++ /dev/null @@ -1,32 +0,0 @@ ---- -name: BRL MM -description: BRL market making research agent. Tracks USDT-BRL spreads, BTC/fiat volume - dynamics, and bot performance for the Brazilian market making operation. -agent_key: ollama:qwen3.5:9 -skills: [] -default_config: - connector_name: binance - quote_currency: BRL -default_trading_context: '' -created_by: 481175164 -created_at: '2026-05-04T00:00:00+00:00' ---- - -### Routines - -#### brl_dashboard -Multi-timeframe technical dashboard for USDT-BRL with candlesticks, SMA, RSI, and support/resistance levels across 4 timeframes (1m, 15m, 1h, 1d). - -#### btc_brl_volume -Compares BTC-USDT vs BTC-BRL volume over 7 days and analyzes 1-second price arbitrage spread to detect opportunities. - -#### btc_fiat_markets -Stacked area chart of BTC trading volume across fiat pairs (BRL, ARS, MXN, EUR, JPY, PLN), normalized to USDT. - -#### bot_report -Comprehensive bot performance report with volume, PnL, fees, market share, rebates, and per-controller breakdown. - -### Journal -- Log market observations: BRL premium/discount trends, volume shifts -- Track bot performance metrics over time -- Note arbitrage windows and their duration From fec6958c5817f24e6798a103ca6056c78ff127ba Mon Sep 17 00:00:00 2001 From: cardosofede Date: Wed, 19 Aug 2026 23:21:26 +0300 Subject: [PATCH 46/57] (feat) apply improvements claude review --- PRIVACY.md | 67 ++++++++++++++++------------ README.md | 9 ++-- condor/telemetry/__init__.py | 32 +++++++------- condor/telemetry/consent.py | 83 ++++++++++++++++++----------------- condor/telemetry/emitter.py | 21 +++++---- condor/telemetry/prompt.py | 32 ++++++-------- condor/telemetry/taps.py | 6 ++- condor/web/routes/settings.py | 15 ++++--- tests/test_telemetry.py | 68 +++++++++++++++++++++------- 9 files changed, 195 insertions(+), 138 deletions(-) diff --git a/PRIVACY.md b/PRIVACY.md index fbb8d2003..31a17d4d1 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -1,31 +1,33 @@ # Privacy Condor is self-hosted. It runs on your machine, holds your exchange API keys, -and places your orders. So the default for anything that could leave that -machine is **off**, and this document is the complete statement of what the one -optional exception does. - -**Short version:** a fresh install sends nothing. It has no telemetry consent -recorded, which resolves to level `off`, at which the emitter returns before it -has looked at its arguments — no buffer, no file, no directory. Nothing changes -until an admin taps "yes" on a prompt. If one does, batches go to the project's -collector at `https://telemetry.hummingbot.org/v1/events`, which is fixed in the -source and cannot be pointed elsewhere; consent is the only switch. +and places your orders. So what leaves that machine by default is the absolute +minimum — an anonymous "this install exists" — and everything beyond that is +opt-in. This document is the complete statement of both. + +**Short version:** a fresh install counts itself and nothing more. With no +consent recorded it runs at level `ping`: a random install id, the version, and +a periodic heartbeat — nothing about you, your users, or your trading. None of +the usage events are sent until an admin taps "yes" on a prompt. Batches go to +the project's collector at `https://telemetry.hummingbot.org/v1/events`, which +is fixed in the source and cannot be pointed elsewhere. The one full kill +switch is `CONDOR_TELEMETRY=off` in the environment. The whole mechanism is about 900 lines in [`condor/telemetry/`](condor/telemetry/). It is meant to be read, not trusted. --- -## What is collected, if you opt in +## What is collected -There are three levels. You choose one; you can change it later. +There are three levels. The consent prompt chooses between `ping` and `usage`; +`off` exists only as an environment override. You can change the answer later. | Level | What it sends | |---|---| -| `off` | Nothing, ever. The emitter is a no-op. **This is the default.** | -| `ping` | Only that this install exists: `install`, `heartbeat`, `version_change`, `shutdown`. | -| `usage` | The above plus the feature, reliability and agent events below. | +| `ping` | Only that this install exists: `install`, `heartbeat`, `version_change`, `shutdown`. **This is the default, and the floor** — the prompt has no "off" option. | +| `usage` | The above plus the feature, reliability and agent events below. Opt-in only. | +| `off` | Nothing, ever. The emitter is a no-op. Reachable only via `CONDOR_TELEMETRY=off` in the environment. | Every batch carries one context block describing the *deployment*, not you: @@ -94,9 +96,10 @@ per-install, the same person on two installs produces two unrelated hashes. ## Where it goes -Nowhere, unless you opt in. +At the default `ping` level, only the four adoption events and the envelope +above. Everything else needs an explicit opt-in. -If you do, batches are POSTed to `https://telemetry.hummingbot.org/v1/events`. +Batches are POSTed to `https://telemetry.hummingbot.org/v1/events`. That address is compiled into [`condor/telemetry/outbox.py`](condor/telemetry/outbox.py) as `COLLECTOR_URL` and is not configurable — there is no environment variable or config key that @@ -115,32 +118,38 @@ You can read exactly what would be sent: cat condor/.runtime/telemetry/outbox.jsonl | jq . ``` -## How to turn it off — or check it is off +## How to change it — or turn it off entirely -It is already off unless you turned it on. To be certain: +Install counting (`ping`) is the floor: it is not an option on the consent +prompt and cannot be disabled from the dashboard, because the project needs an +honest count of installs to know what to support. To check what your install is +doing: ```bash -# The authoritative answer. Prints "off" on a default install. +# The authoritative answer. Prints "ping" on a default install, +# "off" only when CONDOR_TELEMETRY=off is set. uv run python -c "from condor.telemetry import consent; print(consent.level())" ``` Three ways to control it, in order of precedence: -1. **Environment** — `CONDOR_TELEMETRY=off` in your `.env`. This overrides - everything, in both directions, and is the right answer for a headless or - containerized install. -2. **The dashboard API** — `PUT /api/v1/settings/telemetry?level=off` (admin - only). `GET` the same path to see the current state. +1. **Environment** — `CONDOR_TELEMETRY=off` in your `.env`. The one full kill + switch: it overrides everything, suppresses the prompt, and is the right + answer for an install that must send nothing at all. +2. **The dashboard API** — `PUT /api/v1/settings/telemetry?level=ping|usage` + (admin only; `off` is rejected here). `GET` the same path to see the + current state. 3. **`config.yml`** — edit the `telemetry` section directly: ```yaml telemetry: - consent: denied - level: off + consent: granted + level: ping ``` -Turning it off is a **withdrawal, not a pause**: the in-memory buffer and the -outbox file are deleted, so nothing already recorded can be sent afterwards. +Downgrading from `usage` to `ping` is a **withdrawal, not a pause**: the +in-memory buffer and the outbox file are deleted, so nothing already recorded +can be sent afterwards. ## Changes to this document diff --git a/README.md b/README.md index 3544e87f3..57ac83698 100644 --- a/README.md +++ b/README.md @@ -12,10 +12,11 @@ A Telegram bot for monitoring and trading with Hummingbot via the **Hummingbot A > > Full walkthrough: [Securing Condor and Hummingbot API with Tailscale](https://hummingbot.org/blog/posts/securing-condor-and-hummingbot-api-with-tailscale/) · [Hummingbot API Tailscale guide](https://hummingbot.org/hummingbot-api/tailscale/) -> **Privacy:** Condor collects nothing by default. It ships with usage telemetry -> **off**, and it stays that way unless an admin explicitly opts in from the one -> prompt it sends on first boot. What that option would and would not collect — -> where it goes, and how to verify it is off — is spelled out in +> **Privacy:** Condor counts installs anonymously — a random id, the version, +> and a heartbeat; nothing about you or your trading. Everything beyond that is +> **opt-in** from the one prompt it sends on first boot, and +> `CONDOR_TELEMETRY=off` silences telemetry entirely. What each level does and +> does not collect, where it goes, and how to verify it, is spelled out in > [PRIVACY.md](PRIVACY.md). ## Features diff --git a/condor/telemetry/__init__.py b/condor/telemetry/__init__.py index b8aa35c2e..1ba66862c 100644 --- a/condor/telemetry/__init__.py +++ b/condor/telemetry/__init__.py @@ -1,23 +1,24 @@ -"""Anonymous, opt-in usage telemetry (FEAT-023). +"""Anonymous usage telemetry (FEAT-023). -Condor is self-hosted, so the project sees nothing about how installs are used -unless an install chooses to tell it. This package is that channel — and, -because the same process holds exchange API keys, it is built to be auditable in -one sitting rather than to be clever. +Condor is self-hosted, so beyond knowing an install exists the project sees +nothing about how it is used unless the install chooses to tell it. This +package is that channel — and, because the same process holds exchange API +keys, it is built to be auditable in one sitting rather than to be clever. The four facts that define it: -- **Off by default.** A fresh clone has no consent recorded, which resolves to - level ``off``, at which :func:`emit` returns before doing anything at all. No - buffer, no file, no directory. -- **Opt-in, once, by the admin.** One inline-keyboard prompt on boot offers - full usage, install-count-only, or no. The answer is durable and reversible. +- **Every install is counted.** The floor is level ``ping``: the four adoption + events (``install``, ``heartbeat``, ``version_change``, ``shutdown``) and the + anonymous envelope, from the first boot, with no answer required. +- **Usage is opt-in, once, by the admin.** One inline-keyboard prompt on boot + offers full usage or install-count-only. The answer is durable and + reversible. - **Allowlisted.** :mod:`condor.telemetry.schema` declares every event and every property. Anything undeclared is dropped by construction, which is what makes the "never collected" list in ``PRIVACY.md`` a property of the code. -- **Consent is the only switch.** The collector address is fixed in - :mod:`condor.telemetry.outbox`; at level ``off`` nothing is ever recorded, so - there is nothing to send it. +- **The environment is the only kill switch.** The collector address is fixed + in :mod:`condor.telemetry.outbox`; ``CONDOR_TELEMETRY=off`` is the one way to + reach level ``off``, at which nothing is ever recorded. Public surface — call sites should need nothing else:: @@ -63,8 +64,9 @@ def init(hosted: bool = True) -> str: Priming does two things and only two: it resolves the level once so :func:`emit` never has to read the disk on a hot path, and — only if the - level is not ``off`` — it materializes the install's random ids. An install - that never opted in is left exactly as it was found. + level is not ``off`` — it materializes the install's random ids. With the + ping floor that happens on the very first boot; only an install silenced by + ``CONDOR_TELEMETRY=off`` is left exactly as it was found. """ from condor.telemetry import consent, emitter diff --git a/condor/telemetry/consent.py b/condor/telemetry/consent.py index da9935131..8b27db13b 100644 --- a/condor/telemetry/consent.py +++ b/condor/telemetry/consent.py @@ -1,26 +1,25 @@ -"""Consent: the three-state machine that decides whether anything happens at all. +"""Consent: the machine that decides how much an install says. The install — not the individual user — is the unit of consent, because the admin owns the install. The states are ``unknown`` (the default on a fresh -clone), ``granted`` and ``denied``, stored in ``config.yml`` under ``telemetry`` -alongside the install's identity. +clone), ``granted`` and ``denied`` (legacy — older installs could refuse from +the prompt), stored in ``config.yml`` under ``telemetry`` alongside the +install's identity. Two rules matter more than the rest: -**Unknown means off.** Not "buffered until someone decides" — off. An install -whose admin has never answered the prompt collects nothing, holds nothing in -memory, and writes no spool file. The original design buffered pre-consent -events and simply refused to send them; recording data before permission is a -worse default than losing it, so this narrows that decision. Nothing is lost -that we had any right to. +**The floor is ``ping``.** Every install is counted. An install whose admin has +never answered the prompt emits the four adoption events (``install``, +``heartbeat``, ``version_change``, ``shutdown``) and nothing else; the prompt +decides one thing only — whether the ``usage`` events are added on top. There +is no "off" answer on the form. **The environment wins.** ``CONDOR_TELEMETRY`` in ``utils/config.py`` overrides -the stored answer in both directions, so a headless or containerized install is -never blocked on a Telegram prompt, and an operator can force ``off`` no matter -what is on disk. +the stored answer in both directions, and it is the only way to reach level +``off`` — the operator's kill switch for a headless or containerized install. -Reads never create ``config.yml``. On an install that has not been configured at -all, :func:`state` answers ``unknown`` from nothing. +Reads never create ``config.yml``; the identity ids are materialized by +:func:`condor.telemetry.init` on the first boot that can actually emit. """ from __future__ import annotations @@ -41,8 +40,10 @@ GRANTED = "granted" DENIED = "denied" -# Answer -> level, the three buttons of the admin prompt. -ANSWER_LEVELS = {"usage": USAGE, "ping": PING, "off": OFF} +# Answer -> level, the two buttons of the admin prompt. "off" is deliberately +# not an answer: install counting is the floor, and only the environment can +# silence it entirely. +ANSWER_LEVELS = {"usage": USAGE, "ping": PING} _cached_level: str | None = None @@ -113,10 +114,12 @@ def _compute_level() -> str: env = _env_level() if env is not None: return env - if state() != GRANTED: - return OFF - stored = _section().get("level") - return stored if stored in LEVELS else USAGE + if state() == GRANTED: + stored = _section().get("level") + return stored if stored in (PING, USAGE) else USAGE + # Unanswered — and legacy denied — installs are still counted: ping is the + # floor, and only the environment can force `off`. + return PING def refresh() -> str: @@ -172,10 +175,13 @@ def install_secret() -> str: def grant(answer: str) -> str: - """Record a positive answer. ``answer`` is one of :data:`ANSWER_LEVELS`.""" - chosen = ANSWER_LEVELS.get(answer, USAGE) - if chosen == OFF: - return deny() + """Record the admin's answer. ``answer`` is one of :data:`ANSWER_LEVELS`. + + Anything unrecognized — including an "off" tap on a prompt left over from + an older version — lands on ``ping``, the floor, never silently on + ``usage``. + """ + chosen = ANSWER_LEVELS.get(answer, PING) _update( consent=GRANTED, level=chosen, @@ -185,32 +191,29 @@ def grant(answer: str) -> str: return chosen -def deny() -> str: - """Record a refusal and destroy whatever was already on disk. +def _purge_collected() -> None: + """Destroy everything recorded but not yet sent. - Denial deletes the spool and the outbox rather than merely ignoring them: - an answer of "no" should leave nothing behind to be sent by a later bug. + Withdrawing usage consent deletes the spool and the outbox rather than + merely ignoring them: a downgrade should leave nothing behind to be sent + by a later bug. """ - _update( - consent=DENIED, - level=OFF, - decided_at=time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()), - ) from condor.telemetry import emitter, outbox emitter.discard_buffer() outbox.purge() - return OFF def set_level(new_level: str) -> str: - """Change the level of an install that has already consented.""" - if new_level not in LEVELS: + """Change the level of an install. ``off`` is not accepted here: the floor + is ``ping``, and only ``CONDOR_TELEMETRY=off`` reaches ``off``.""" + if new_level not in (PING, USAGE): return level() - if new_level == OFF: - return deny() + downgrading = new_level == PING and level() == USAGE _update(consent=GRANTED, level=new_level) ensure_identity() + if downgrading: + _purge_collected() return new_level @@ -220,8 +223,8 @@ def set_level(new_level: str) -> str: def mark_feature_seen(feature: str) -> bool: """True the first time this install ever uses ``feature``, False after. - Backs the ``feature_first_use`` activation funnel. Only called when - telemetry is on, so an opted-out install never accumulates the list. + Backs the ``feature_first_use`` activation funnel. Only called at level + ``usage``, so an install that has not opted in never accumulates the list. """ section = _section() seen = list(section.get("features_seen") or []) diff --git a/condor/telemetry/emitter.py b/condor/telemetry/emitter.py index aa84890ed..5010ab870 100644 --- a/condor/telemetry/emitter.py +++ b/condor/telemetry/emitter.py @@ -8,9 +8,11 @@ the fallback is silence. 2. **It never blocks and never does I/O** in the host process. It appends to a bounded ``deque``; the network is somebody else's job, 15 minutes later. -3. **It is off unless someone said yes.** The first gate is a cached level read, - and on an install that has not opted in that read returns ``off`` and the - function returns before it has looked at its own arguments. +3. **It says no more than the level allows.** The first gate is a cached level + read: ``CONDOR_TELEMETRY=off`` makes it return before it has looked at its + own arguments, and at the default ``ping`` floor only the four adoption + events pass the schema gate — everything else needs an explicit ``usage`` + opt-in. 4. **It cannot emit what the schema does not declare.** Properties are allowlisted, type-checked and truncated by :mod:`condor.telemetry.schema`. @@ -123,7 +125,8 @@ def drain() -> tuple[list[dict], int]: def discard_buffer() -> None: - """Throw the ring away without sending it. Used when consent is denied.""" + """Throw the ring away without sending it. Used when usage consent is + withdrawn.""" global _dropped _buffer.clear() _dropped = 0 @@ -133,14 +136,16 @@ def discard_buffer() -> None: async def flush(reason: str = "job") -> int: """Try to deliver everything pending. Returns the number of events sent. - Returns 0 without reading a file when consent is not granted, and 0 without - touching the network when no endpoint is configured — the shipped state, in - which events simply accumulate in the capped outbox. + Returns 0 without reading a file when the operator has forced ``off``, and + 0 without touching the network when no endpoint is configured — the shipped + state, in which events simply accumulate in the capped outbox. An + unanswered install flushes too: the ping floor is only worth anything if + the adoption events actually leave. """ try: from condor.telemetry import consent, context, outbox - if consent.state() != consent.GRANTED or consent.level() == consent.OFF: + if consent.level() == consent.OFF: return 0 events, dropped_count = drain() diff --git a/condor/telemetry/prompt.py b/condor/telemetry/prompt.py index dd3ee0957..8d9ec4613 100644 --- a/condor/telemetry/prompt.py +++ b/condor/telemetry/prompt.py @@ -1,10 +1,10 @@ """The one-tap consent prompt, and the callback that answers it. -Opt-in only works if asking is cheap, so this is a single message with three +Opt-in only works if asking is cheap, so this is a single message with two buttons next to the "Condor is online" notification the admin already gets. It is sent at most once per version, the intent is written to disk *before* the message goes out (a crash loop must not re-ask forever), and until it is -answered the install emits nothing at all. +answered the install stays at the ``ping`` floor — counted, nothing more. """ from __future__ import annotations @@ -17,14 +17,17 @@ _TEXT = ( "Help improve Condor?\n\n" - "Condor can send an anonymous, allowlisted usage summary to the project: " - "which commands and screens get used, what breaks, and which models agents " - "run. It is off right now and stays off unless you say yes.\n\n" + "Condor counts installs so the project knows it is used: a random id, the " + "version, and an uptime ping — nothing about you or your trading. That is " + "always on.\n\n" + "It can also send an anonymous, allowlisted usage summary: which commands " + "and screens get used, what breaks, and which models agents run. That part " + "is up to you.\n\n" "Never included: API keys, wallet addresses, server names or URLs, " "trading pairs, amounts, balances, positions, prompts or agent replies, " "and no Telegram id or username.\n\n" "Full details in PRIVACY.md at the root of the repo, which also says how " - "to change or withdraw this answer at any time." + "to change this answer at any time." ) @@ -35,7 +38,8 @@ def keyboard(): [ [ InlineKeyboardButton( - "Yes, help improve Condor", callback_data=f"{CALLBACK_PREFIX}:usage" + "Yes, share usage summaries", + callback_data=f"{CALLBACK_PREFIX}:usage", ) ], [ @@ -43,7 +47,6 @@ def keyboard(): "Only count my install", callback_data=f"{CALLBACK_PREFIX}:ping" ) ], - [InlineKeyboardButton("No thanks", callback_data=f"{CALLBACK_PREFIX}:off")], ] ) @@ -98,14 +101,8 @@ async def callback_handler(update, context) -> None: answer = ( (query.data or "").split(":", 1)[1] if ":" in (query.data or "") else "" ) - if answer == "off": - consent.deny() - await query.edit_message_text( - "Telemetry stays off. Nothing was collected, and the local " - "buffer has been deleted." - ) - return - + # An "off" tap can only come from a prompt sent by an older version; + # grant() lands anything unrecognized on ping, the floor. chosen = consent.grant(answer) from condor.telemetry import emitter @@ -113,8 +110,7 @@ async def callback_handler(update, context) -> None: if chosen == consent.PING: await query.edit_message_text( "Thanks. Condor will only report that this install exists and " - "which version it runs. PRIVACY.md says how to change or " - "withdraw this." + "which version it runs. PRIVACY.md says how to change this." ) else: await query.edit_message_text( diff --git a/condor/telemetry/taps.py b/condor/telemetry/taps.py index c32117e55..dac85057c 100644 --- a/condor/telemetry/taps.py +++ b/condor/telemetry/taps.py @@ -94,10 +94,12 @@ def _note_user(hashed: str | None, surface: str) -> None: @_quiet def feature_first_use(feature: str) -> None: """Fire once per install, ever. The activation funnel.""" - if not _on(): - return from condor.telemetry import consent + # Gated on `usage`, not merely "on": at the ping floor the event could + # never be sent, and the features_seen list should not accumulate for it. + if consent.level() != consent.USAGE: + return if consent.mark_feature_seen(feature): emit("feature_first_use", feature=feature) diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index b2d1709d2..66fc57cc4 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -902,13 +902,15 @@ async def get_telemetry_settings(user: WebUser = Depends(get_current_user)): @router.put("/telemetry") async def set_telemetry_settings( - level: str = Query(..., description="off | ping | usage"), + level: str = Query(..., description="ping | usage"), user: WebUser = Depends(get_current_user), ): """Change the install's telemetry level. Admin only, and reversible. - Setting ``off`` is a withdrawal, not a pause: the buffer and the outbox are - deleted, so nothing already recorded can be sent later. + ``ping`` is the floor — install counting cannot be turned off here; only + ``CONDOR_TELEMETRY=off`` in the environment silences telemetry entirely. + Downgrading from ``usage`` is a withdrawal, not a pause: the buffer and the + outbox are deleted, so nothing already recorded can be sent later. """ from condor.telemetry import consent @@ -918,9 +920,12 @@ async def set_telemetry_settings( status_code=403, detail="Telemetry is an install-wide setting; only the admin can change it", ) - if level not in consent.LEVELS: + if level not in (consent.PING, consent.USAGE): raise HTTPException( - status_code=400, detail=f"level must be one of {', '.join(consent.LEVELS)}" + status_code=400, + detail="level must be ping or usage; install counting cannot be " + "turned off here — set CONDOR_TELEMETRY=off in the environment to " + "disable telemetry entirely", ) if consent.env_overridden(): raise HTTPException( diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 0cf2ef8f9..32a016f40 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -45,23 +45,24 @@ def _grant(level="usage"): consent.grant(level) -# ── The default is silence ─────────────────────────────────────────────── +# ── The default is the ping floor ──────────────────────────────────────── -def test_a_fresh_install_is_off_and_stays_off(install): - """No consent recorded means no telemetry — not "buffered until asked".""" +def test_a_fresh_install_pings_and_says_nothing_more(install): + """No consent recorded means install counting only — the ping floor.""" assert consent.state() == consent.UNKNOWN - assert consent.level() == consent.OFF + assert consent.level() == consent.PING for name in schema.EVENTS: emitter.emit(name, surface="telegram") - assert emitter.buffered() == 0 - assert emitter.dropped() == 0 + names = {e["name"] for e in emitter.drain()[0]} + assert names == set(schema.PING_EVENTS) -def test_nothing_is_written_to_disk_when_off(install): - """Acceptance criterion: no condor/.runtime/telemetry directory is created.""" +def test_usage_events_write_nothing_to_disk_at_the_ping_floor(install): + """Acceptance criterion: no condor/.runtime/telemetry directory is created + by activity that the install has no consent to report.""" emitter.set_hosted(False) # the spooling path, which is the one that does I/O emitter.emit("command", name="portfolio", surface="telegram") assert not outbox.root().exists() @@ -83,7 +84,7 @@ def test_env_off_overrides_a_granted_consent(install, monkeypatch): def test_a_nonsense_env_level_is_ignored_not_obeyed(install, monkeypatch): monkeypatch.setattr("utils.config.CONDOR_TELEMETRY", "yes-please", raising=False) consent.refresh() - assert consent.level() == consent.OFF + assert consent.level() == consent.PING # the floor, not whatever was typed def test_ping_level_sends_only_the_adoption_events(install): @@ -99,8 +100,8 @@ def test_ping_level_sends_only_the_adoption_events(install): # ── Nothing transmits ──────────────────────────────────────────────────── -def test_flush_is_a_no_op_without_consent(install, monkeypatch): - """A full session of activity must produce zero outbound requests.""" +def test_usage_activity_never_transmits_without_consent(install, monkeypatch): + """A full session of activity must produce zero outbound usage events.""" calls = [] monkeypatch.setattr(outbox, "post", lambda env: calls.append(env)) @@ -113,6 +114,25 @@ def test_flush_is_a_no_op_without_consent(install, monkeypatch): assert not outbox.root().exists() +def test_the_ping_floor_actually_flushes_without_an_answer(install, monkeypatch): + """An unanswered install still delivers its adoption events — the count is + only worth anything if it leaves the machine.""" + sent = [] + + async def ok_post(envelope): + sent.append(envelope) + return True + + monkeypatch.setattr(outbox, "endpoint", lambda: "http://collector.invalid") + monkeypatch.setattr(outbox, "post", ok_post) + + assert consent.state() == consent.UNKNOWN + emitter.emit("install") + assert asyncio.run(emitter.flush("test")) == 1 + assert [e["name"] for env in sent for e in env["events"]] == ["install"] + assert sent[0]["level"] == consent.PING + + def test_the_endpoint_is_fixed_and_not_configurable(install, monkeypatch): """The collector address is compiled in; no env var may redirect it.""" monkeypatch.setenv("CONDOR_TELEMETRY_URL", "http://attacker.invalid/v1/events") @@ -171,7 +191,7 @@ def test_the_outbox_respects_its_cap(install): assert len(outbox.take_stashed()) == outbox.MAX_OUTBOX_EVENTS -def test_denying_consent_destroys_what_was_collected(install): +def test_withdrawing_usage_consent_destroys_what_was_collected(install): _grant("usage") emitter.emit("command", name="portfolio", surface="telegram") outbox.stash( @@ -185,13 +205,25 @@ def test_denying_consent_destroys_what_was_collected(install): ) assert outbox.outbox_path().exists() - consent.deny() + consent.set_level("ping") - assert consent.level() == consent.OFF + assert consent.level() == consent.PING assert emitter.buffered() == 0 assert not outbox.outbox_path().exists() +def test_off_is_not_a_grantable_answer(install): + """The consent form has no "off": a stale or unrecognized answer lands on + ping, the floor, and set_level rejects "off" outright.""" + from config_manager import get_config_manager + + get_config_manager() # materialize config.yml in the tmp cwd + assert consent.grant("off") == consent.PING + assert consent.state() == consent.GRANTED + assert consent.set_level("off") == consent.PING # rejected, level unchanged + assert consent.level() == consent.PING + + # ── The leak test ──────────────────────────────────────────────────────── FORBIDDEN = { @@ -435,8 +467,9 @@ def test_emit_costs_under_50_microseconds_on_the_hot_path(install): assert per_call < 50e-6, f"emit() took {per_call * 1e6:.1f}us" -def test_emit_is_free_when_off(install): - """The gate a trading path actually pays for.""" +def test_emit_is_free_below_usage(install): + """The gate a trading path actually pays for: at the ping floor a usage + event is rejected by the schema gate before any work happens.""" iterations = 2000 started = time.perf_counter() for _ in range(iterations): @@ -498,7 +531,8 @@ def test_a_callback_reports_module_and_verb_only(install): def test_the_telegram_tap_emits_nothing_before_consent(install): - """The acceptance criterion, at the seam rather than at the emitter.""" + """The acceptance criterion, at the seam rather than at the emitter: + commands and callbacks are usage events, and the ping floor drops them.""" assert consent.state() == consent.UNKNOWN for update in ( _FakeUpdate(text="/portfolio"), From 3726ed8adc4cfe44a76b81c04c6c6ddf05531674 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Aug 2026 01:56:33 +0300 Subject: [PATCH 47/57] (feat) add Gateway token registration button to pool banner: retry failed saves and replace ticker collisions in place --- condor/fetchers/gateway_tokens.py | 40 ++++ condor/web/routes/dex.py | 90 +++++++++ frontend/src/lib/api.ts | 28 +++ frontend/src/pages/DexPool.tsx | 111 ++++++++++- tests/test_gateway_tokens.py | 320 ++++++++++++++++++++++++++++++ 5 files changed, 581 insertions(+), 8 deletions(-) create mode 100644 tests/test_gateway_tokens.py diff --git a/condor/fetchers/gateway_tokens.py b/condor/fetchers/gateway_tokens.py index cc9a210f6..5b8af68b3 100644 --- a/condor/fetchers/gateway_tokens.py +++ b/condor/fetchers/gateway_tokens.py @@ -43,6 +43,16 @@ def reset_listed_memo() -> None: _listed.clear() +def forget_listed(network_id: str, address: str) -> None: + """Drop one confirmed listing, after its token is deleted from the list. + + Without this, replacing a ticker's holder would leave the deleted address + memoized as listed — and every pool that uses it would trust a balance + Gateway can no longer read. + """ + _listed.discard(_memo_key(network_id, address)) + + def _memo_key(network_id: str, address: str) -> tuple[str, str]: return (network_id, address.lower()) @@ -92,6 +102,36 @@ async def _is_listed(client: Any, network_id: str, address: str) -> bool: ) +async def find_symbol_holder( + client: Any, network_id: str, symbol: str, exclude: str | None = None +) -> dict[str, Any] | None: + """The token on ``network_id``'s list that holds ``symbol``, if any. + + This is the other half of a ``symbol_taken`` verdict: the verdict says the + ticker is in use, this says by *whom* — which is what the user needs to + decide whether the holder should be replaced. ``exclude`` skips the token + being registered itself, so a half-written state cannot name it as its own + conflict. + """ + response = await client.gateway.get_network_tokens(network_id, search=symbol) + tokens = response.get("tokens", []) if isinstance(response, dict) else response + wanted = symbol.lower() + excluded = (exclude or "").lower() + for token in tokens or []: + entry = token or {} + if str(entry.get("symbol") or "").lower() != wanted: + continue + address = str(entry.get("address") or "") + if address.lower() == excluded: + continue + return { + "symbol": entry.get("symbol"), + "address": address, + "name": entry.get("name"), + } + return None + + async def ensure_tokens_listed( client: Any, network_id: str, addresses: Iterable[str] ) -> dict[str, str]: diff --git a/condor/web/routes/dex.py b/condor/web/routes/dex.py index 19cea6b57..bba17b46f 100644 --- a/condor/web/routes/dex.py +++ b/condor/web/routes/dex.py @@ -371,6 +371,96 @@ async def ensure_dex_tokens( return {"tokens": verdicts} +class AddTokenRequest(BaseModel): + """One token the user explicitly asked to put on Gateway's list.""" + + network: str = Field(description="Gateway network id, e.g. solana-mainnet-beta") + address: str = Field(description="Token mint/contract to register") + symbol: str | None = Field( + default=None, + description="Ticker the pool shows for it, used to name a conflicting holder", + ) + replace: bool = Field( + default=False, + description="Delete the token currently holding the ticker before adding", + ) + + +@router.post("/servers/{name}/dex/tokens/add") +async def add_dex_token( + name: str, + body: AddTokenRequest, + user: WebUser = Depends(require_server_access), +): + """Register one token because the user clicked a button asking for it. + + The automatic path (``/dex/tokens``, above) reports rather than forces: a + ``failed`` verdict is a Gateway hiccup, and ``symbol_taken`` means another + token already owns the ticker — a conflict only a human should resolve, + because deleting the holder changes what that ticker means everywhere. + This route is that human's answer. Without ``replace`` it retries the + registration and, on a collision, names the holder so the browser can ask; + with ``replace`` it deletes the holder first and registers the new token in + its place. + + Unlike the ensure route, an upstream error here is raised, not folded into + a verdict: the user pressed a button and deserves the actual reason. + """ + from condor.fetchers.gateway_tokens import ( + ensure_tokens_listed, + find_symbol_holder, + forget_listed, + token_addresses, + ) + from condor.pool_data import GECKO_TO_GATEWAY_NETWORK, NETWORK_TO_GECKO + + gateway_network = GECKO_TO_GATEWAY_NETWORK.get( + NETWORK_TO_GECKO.get(body.network, "") + ) + if not gateway_network: + raise HTTPException(status_code=400, detail="Unknown network") + + addresses = token_addresses(body.address) + if not addresses: + raise HTTPException(status_code=400, detail="Not a token address") + address = addresses[0] + + cm = get_config_manager() + client = await cm.get_client(name) + try: + if body.replace and body.symbol: + holder = await find_symbol_holder( + client, gateway_network, body.symbol, exclude=address + ) + if holder: + await client.gateway.delete_token(gateway_network, holder["address"]) + forget_listed(gateway_network, holder["address"]) + + verdicts = await ensure_tokens_listed(client, gateway_network, [address]) + verdict = verdicts.get(address, "failed") + + # The collision's other half: who holds the ticker. Looked up only when + # the answer is needed, and only nameable when the pool knows its own + # ticker — a `???` pool gets the verdict without the culprit. + conflict = None + if verdict == "symbol_taken" and body.symbol: + conflict = await find_symbol_holder( + client, gateway_network, body.symbol, exclude=address + ) + except Exception as e: + logger.warning("add token %s on %s failed: %s", address, gateway_network, e) + raise HTTPException(status_code=502, detail=f"Gateway refused: {e}") from e + + # `ensure_tokens_listed` folds its own upstream errors into this verdict so + # the automatic path stays quiet; a clicked button must not. + if verdict == "failed": + raise HTTPException( + status_code=502, + detail="Gateway could not register the token — is it running?", + ) + return {"verdict": verdict, "conflict": conflict} + + @router.get("/servers/{name}/dex/pools-by-address") async def list_pools_by_address( name: str, diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 65fc98b65..85f3d2613 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -389,6 +389,13 @@ export interface DexUpstream { */ export type DexTokenVerdict = "listed" | "added" | "symbol_taken" | "failed"; +/** The token that already holds a ticker a new token wanted. */ +export interface DexTokenConflict { + symbol: string; + address: string; + name?: string | null; +} + /** A page of pools. `has_more`, not a count — no upstream reports a total. */ export interface PoolPage { pools: PoolSummary[]; @@ -1474,6 +1481,27 @@ export const api = { { method: "POST", body: JSON.stringify({ network, addresses }) }, ), + /** + * Register one token because the user asked to, where the automatic ensure + * only reported. Retries a `failed` verdict outright; on `symbol_taken` the + * answer names the ticker's current holder, and a second call with + * `replace: true` deletes that holder and registers this token in its place. + */ + addDexToken: ( + server: string, + network: string, + address: string, + symbol?: string, + replace?: boolean, + ) => + apiFetch<{ verdict: DexTokenVerdict; conflict: DexTokenConflict | null }>( + `/api/v1/servers/${encodeURIComponent(server)}/dex/tokens/add`, + { + method: "POST", + body: JSON.stringify({ network, address, symbol, replace }), + }, + ), + /** One pool by address, so /dex/{network}/{address} renders from the URL alone. */ getDexPoolByAddress: (server: string, address: string, network: string) => apiFetch( diff --git a/frontend/src/pages/DexPool.tsx b/frontend/src/pages/DexPool.tsx index b2bd02265..173568ad0 100644 --- a/frontend/src/pages/DexPool.tsx +++ b/frontend/src/pages/DexPool.tsx @@ -31,7 +31,7 @@ import { usePairBalances } from "@/hooks/usePairBalances"; import { useResizeDrag } from "@/hooks/useResizeDrag"; import { useServer } from "@/hooks/useServer"; import { useCondorWebSocket } from "@/hooks/useWebSocket"; -import { api } from "@/lib/api"; +import { api, type DexTokenConflict } from "@/lib/api"; import { connectorCapabilities } from "@/lib/connector-capabilities"; import { LOOKBACK_OPTIONS } from "@/lib/gridExecutor"; @@ -151,15 +151,57 @@ export function DexPool() { // Reported rather than swallowed: Gateway refuses to list a token whose ticker // another token already holds, and the only visible symptom is a balance that - // stays 0 for a wallet that is not empty. + // stays 0 for a wallet that is not empty. Each entry keeps its ticker so the + // add button can say what it adds — and so the server can name a collision. const unlistedTokens = useMemo( () => Object.entries(tokenVerdicts ?? {}) .filter(([, verdict]) => verdict === "symbol_taken" || verdict === "failed") - .map(([address]) => address), - [tokenVerdicts], + .map(([tokenAddress]) => { + const ticker = + tokenAddress === pool?.base_token_address + ? pool?.base_symbol + : tokenAddress === pool?.quote_token_address + ? pool?.quote_symbol + : undefined; + return { + address: tokenAddress, + symbol: ticker && ticker !== "???" ? ticker : undefined, + }; + }), + [tokenVerdicts, pool], ); + // The collision the last add ran into: who holds the ticker, so the banner + // can ask "replace it?" instead of failing with the same verdict again. + const [tokenConflict, setTokenConflict] = useState<{ + address: string; + symbol?: string; + holder: DexTokenConflict; + } | null>(null); + + const addTokenMutation = useMutation({ + mutationFn: (vars: { address: string; symbol?: string; replace?: boolean }) => + api.addDexToken(server!, network, vars.address, vars.symbol, vars.replace), + onSuccess: (data, vars) => { + if (data.verdict === "added" || data.verdict === "listed") { + setTokenConflict(null); + // Re-runs the ensure (its verdicts now read `listed`, dropping the + // banner) and refetches the portfolio the balance panels read from. + queryClient.invalidateQueries({ + queryKey: ["dex-ensure-tokens", server, network, pool?.address], + }); + queryClient.invalidateQueries({ queryKey: ["portfolio", server] }); + } else if (data.verdict === "symbol_taken" && data.conflict) { + setTokenConflict({ + address: vars.address, + symbol: vars.symbol, + holder: data.conflict, + }); + } + }, + }); + const { data: venues = [] } = useQuery({ queryKey: ["venues", server], queryFn: () => api.getVenues(server!), @@ -347,7 +389,8 @@ export function DexPool() { {/* Gateway lists a token by ticker, so a pool whose token shares a ticker with one already on the list cannot be registered — and the only symptom - is a balance that reads 0 for a wallet that is not empty. */} + is a balance that reads 0 for a wallet that is not empty. The button is + the fix in place: retry a failed save, or replace the ticker's holder. */} {unlistedTokens.length > 0 && (
- Balances for {unlistedTokens.map((a) => `${a.slice(0, 6)}…`).join(", ")}{" "} - will read 0 — add {unlistedTokens.length === 1 ? "it" : "them"} under - Gateway tokens to size orders from this wallet. + Balances for{" "} + {unlistedTokens + .map((t) => t.symbol ?? `${t.address.slice(0, 6)}…`) + .join(", ")}{" "} + will read 0 until {unlistedTokens.length === 1 ? "it is" : "they are"}{" "} + added. + {unlistedTokens.map((t) => ( + + ))} + {tokenConflict && ( + + {tokenConflict.holder.symbol} already names{" "} + {tokenConflict.holder.address.slice(0, 6)}… on the list — replace + it with this pool's token? + + + )} + {!tokenConflict && + addTokenMutation.data?.verdict === "symbol_taken" && + !addTokenMutation.data.conflict && ( + + Its ticker is already used by another token on the list — resolve + it under Gateway tokens. + + )} + {addTokenMutation.isError && ( + + {addTokenMutation.error instanceof Error + ? addTokenMutation.error.message + : "Add failed"} + + )}
)} diff --git a/tests/test_gateway_tokens.py b/tests/test_gateway_tokens.py new file mode 100644 index 000000000..c525bece3 --- /dev/null +++ b/tests/test_gateway_tokens.py @@ -0,0 +1,320 @@ +"""A pool's tokens reach Gateway's token list on their own. + +Gateway resolves a swap by mint address, but not a balance: it skips token +accounts whose mint is not on the network's token list, so an unregistered token +reads as 0 and every panel that sizes an order off a balance is working from an +empty wallet. Registration therefore has to happen without anyone asking for it. + +Three shapes are pinned here. That an already-listed token is never re-saved (the +memo is what makes opening a pool twice free). That a save is *verified* by +re-reading the list, because Gateway answers 200 and writes nothing when another +token already holds the ticker — the one failure that stays silent otherwise. And +that a failure is never remembered, so it stays retryable on the next visit. +""" + +import asyncio + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.dex as dex_routes +from condor.fetchers import gateway_tokens +from condor.fetchers.gateway_tokens import ensure_tokens_listed, token_addresses +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +MINT = "9QFfgxdSqH5zT7j6rZb1y6SZhw2aFtcQu2r6BuYpump" +USDC = "EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v" +NETWORK = "solana-mainnet-beta" + + +class FakeGateway: + """Gateway's token list, with the two behaviours that matter kept honest.""" + + def __init__(self, listed=(), refuse_save=False, fail_search=False): + self.tokens = [{"address": a, "symbol": a[:4]} for a in listed] + self.refuse_save = refuse_save + self.fail_search = fail_search + self.searches: list[str] = [] + self.saves: list[str] = [] + self.deletes: list[str] = [] + + async def get_network_tokens(self, network_id, search=None): + if self.fail_search: + raise RuntimeError("gateway down") + self.searches.append(search) + # Gateway matches search against symbol, name and address alike. + needle = (search or "").lower() + return { + "tokens": [ + t + for t in self.tokens + if not needle + or needle in str(t["address"]).lower() + or needle in str(t.get("symbol") or "").lower() + ] + } + + async def save_network_token(self, network_id, token_address): + self.saves.append(token_address) + # A ticker already held by another token: Gateway answers 200 and keeps + # the list unchanged. + if self.refuse_save: + return {"message": "already exists"} + self.tokens.append({"address": token_address, "symbol": "NEW"}) + return {"message": "added"} + + async def delete_token(self, network_id, token_address): + self.deletes.append(token_address) + before = len(self.tokens) + self.tokens = [t for t in self.tokens if t["address"] != token_address] + # The holder is deleted, so its ticker no longer blocks a save. + if len(self.tokens) < before: + self.refuse_save = False + return {"message": "deleted"} + + +class FakeClient: + def __init__(self, gateway): + self.gateway = gateway + + +@pytest.fixture(autouse=True) +def _clean_memo(): + gateway_tokens.reset_listed_memo() + yield + gateway_tokens.reset_listed_memo() + + +def test_a_missing_token_is_registered_without_being_asked_for(): + gw = FakeGateway(listed=[USDC]) + verdicts = asyncio.run(ensure_tokens_listed(FakeClient(gw), NETWORK, [MINT, USDC])) + + assert verdicts == {MINT: "added", USDC: "listed"} + assert gw.saves == [MINT] + + +def test_a_known_token_is_never_saved_twice(): + gw = FakeGateway(listed=[]) + client = FakeClient(gw) + + asyncio.run(ensure_tokens_listed(client, NETWORK, [MINT])) + second = asyncio.run(ensure_tokens_listed(client, NETWORK, [MINT])) + + assert second == {MINT: "listed"} + # One save, and no second lookup: the memo answers the repeat visit outright. + assert gw.saves == [MINT] + assert gw.searches.count(MINT) == 2 # the pre-check and the verification + + +def test_a_save_that_wrote_nothing_is_reported_not_assumed(): + # Gateway refuses a token whose ticker another token already holds — with a + # 200. Trusting the response would leave a token silently unlisted, and its + # balance silently 0. + gw = FakeGateway(listed=[], refuse_save=True) + verdicts = asyncio.run(ensure_tokens_listed(FakeClient(gw), NETWORK, [MINT])) + + assert verdicts == {MINT: "symbol_taken"} + + +def test_a_failed_lookup_stays_retryable(): + gw = FakeGateway(fail_search=True) + client = FakeClient(gw) + + assert asyncio.run(ensure_tokens_listed(client, NETWORK, [MINT])) == { + MINT: "failed" + } + + # Nothing was remembered, so a later visit tries again rather than trusting + # a state that was never confirmed. + gw.fail_search = False + assert asyncio.run(ensure_tokens_listed(client, NETWORK, [MINT])) == {MINT: "added"} + + +def test_only_address_shaped_values_are_sent_upstream(): + # A DEX pair is `-`: splitting it yields one thing + # that can be registered and one that cannot. + assert token_addresses(f"{MINT}-USDC") == [MINT] + assert token_addresses(MINT, MINT) == [MINT] # deduped + assert token_addresses("SOL-USDC", None, "") == [] + + +# --------------------------------------------------------------------------- +# The route the pool workspace calls on open. +# --------------------------------------------------------------------------- + +USER = WebUser(id=222, username="u", first_name="U", role="user") + + +class FakeConfigManager: + def __init__(self, gateway): + self.client = FakeClient(gateway) + + def has_server_access(self, user_id, name, *a, **kw): + return True + + async def get_client(self, name): + return self.client + + +@pytest.fixture +def route_client(monkeypatch): + gw = FakeGateway(listed=[USDC]) + cm = FakeConfigManager(gw) + monkeypatch.setattr(dex_routes, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + app = FastAPI() + app.include_router(dex_routes.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app), gw + + +def test_opening_a_pool_registers_both_of_its_tokens(route_client): + client, gw = route_client + + body = client.post( + "/servers/srv/dex/tokens", + json={"network": "solana-mainnet-beta", "addresses": [MINT, USDC]}, + ).json() + + assert body["tokens"] == {MINT: "added", USDC: "listed"} + assert gw.saves == [MINT] + + +def test_a_chain_named_the_gecko_way_still_reaches_gateway(route_client): + # Pool rows carry a chain id from whichever upstream produced them; Gateway + # only answers to its own spelling, so the route canonicalizes rather than + # rejecting a pool the user can plainly see. + client, _gw = route_client + + response = client.post( + "/servers/srv/dex/tokens", json={"network": "solana", "addresses": [MINT]} + ) + + assert response.status_code == 200 + assert response.json()["tokens"] == {MINT: "added"} + + +def test_a_chain_gateway_does_not_reach_is_refused(route_client): + client, _gw = route_client + + response = client.post( + "/servers/srv/dex/tokens", json={"network": "not-a-chain", "addresses": [MINT]} + ) + + assert response.status_code == 400 + + +def test_a_ticker_is_not_sent_upstream_as_an_address(route_client): + # `USDC` is a quote symbol, not something Gateway can look up by address. + client, gw = route_client + + body = client.post( + "/servers/srv/dex/tokens", + json={"network": "solana-mainnet-beta", "addresses": ["USDC", ""]}, + ).json() + + assert body["tokens"] == {} + assert gw.saves == [] + + +# --------------------------------------------------------------------------- +# The banner's add button: /dex/tokens/add, the human's answer to a verdict. +# --------------------------------------------------------------------------- + + +@pytest.fixture +def make_client(monkeypatch): + def _make(gw): + cm = FakeConfigManager(gw) + monkeypatch.setattr(dex_routes, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + app = FastAPI() + app.include_router(dex_routes.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app) + + return _make + + +def test_the_add_button_registers_what_the_ensure_could_not(make_client): + client = make_client(FakeGateway(listed=[])) + + body = client.post( + "/servers/srv/dex/tokens/add", json={"network": NETWORK, "address": MINT} + ).json() + + assert body["verdict"] == "added" + + +def test_a_ticker_collision_names_its_holder(make_client): + # The verdict alone says the ticker is in use; the button's answer says by + # whom, which is what the user needs to decide whether to replace it. + gw = FakeGateway(refuse_save=True) + gw.tokens = [{"address": USDC, "symbol": "PUMP", "name": "Other Pump"}] + client = make_client(gw) + + body = client.post( + "/servers/srv/dex/tokens/add", + json={"network": NETWORK, "address": MINT, "symbol": "PUMP"}, + ).json() + + assert body["verdict"] == "symbol_taken" + assert body["conflict"]["address"] == USDC + assert gw.deletes == [] # naming the holder must not touch it + + +def test_replace_deletes_the_holder_then_registers(make_client): + gw = FakeGateway(refuse_save=True) + gw.tokens = [{"address": USDC, "symbol": "PUMP"}] + client = make_client(gw) + + body = client.post( + "/servers/srv/dex/tokens/add", + json={"network": NETWORK, "address": MINT, "symbol": "PUMP", "replace": True}, + ).json() + + assert gw.deletes == [USDC] + assert body["verdict"] == "added" + + +def test_a_replaced_holder_is_forgotten_by_the_memo(make_client): + # USDC was seen on the list and memoized; after a replace deletes it, a later + # ensure must re-read the list rather than trust a listing that is gone. + gw = FakeGateway(refuse_save=True) + gw.tokens = [{"address": USDC, "symbol": "PUMP"}] + client = make_client(gw) + assert asyncio.run(ensure_tokens_listed(FakeClient(gw), NETWORK, [USDC])) == { + USDC: "listed" + } + + client.post( + "/servers/srv/dex/tokens/add", + json={"network": NETWORK, "address": MINT, "symbol": "PUMP", "replace": True}, + ) + + verdicts = asyncio.run(ensure_tokens_listed(FakeClient(gw), NETWORK, [USDC])) + assert verdicts == {USDC: "added"} # re-registered, not answered from memory + + +def test_the_button_surfaces_a_gateway_error_instead_of_a_verdict(make_client): + # The automatic ensure folds errors into `failed`; a clicked button deserves + # the actual reason. + client = make_client(FakeGateway(fail_search=True)) + + response = client.post( + "/servers/srv/dex/tokens/add", json={"network": NETWORK, "address": MINT} + ) + + assert response.status_code == 502 + + +def test_the_button_refuses_a_value_that_is_not_an_address(make_client): + client = make_client(FakeGateway()) + + response = client.post( + "/servers/srv/dex/tokens/add", json={"network": NETWORK, "address": "USDC"} + ) + + assert response.status_code == 400 From 2340c0840458ee7c146abc75b6a85b705c5407f4 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Aug 2026 09:35:21 +0300 Subject: [PATCH 48/57] (fix) label LP chart lines as Upper/Lower/Upper limit instead of grid's Start/End/Limit --- frontend/src/components/executor/lp-config.ts | 10 ++++++++++ frontend/src/components/executor/types.ts | 7 +++++++ frontend/src/components/trade/TradeChart.tsx | 18 ++++++++++++------ frontend/src/pages/CreateExecutor.tsx | 1 + frontend/src/pages/DexPool.tsx | 1 + 5 files changed, 31 insertions(+), 6 deletions(-) diff --git a/frontend/src/components/executor/lp-config.ts b/frontend/src/components/executor/lp-config.ts index f7324196f..247a83a43 100644 --- a/frontend/src/components/executor/lp-config.ts +++ b/frontend/src/components/executor/lp-config.ts @@ -313,6 +313,15 @@ const SLOT_FIELD: Record = { limit2: "lower_limit_price", }; +// The slots are named for the grid executor; on an LP range they are prices, +// not a direction of travel, so the chart is told what they mean here. +const LP_LINE_LABELS: Partial> = { + start: "Upper", + end: "Lower", + limit: "Upper limit", + limit2: "Lower limit", +}; + export function isMeteoraProvider(provider: string): boolean { return provider.toLowerCase().startsWith("meteora/"); } @@ -349,6 +358,7 @@ export function useLpConfig( side: state.side === LP_SIDE_SELL ? 2 : 1, minSpread: 0, activePickField: PICK_SLOT[state.activePickField ?? ""] ?? null, + lineLabels: LP_LINE_LABELS, extraLines: state.lower_limit_price > 0 ? [ diff --git a/frontend/src/components/executor/types.ts b/frontend/src/components/executor/types.ts index ea549dadc..aadb46804 100644 --- a/frontend/src/components/executor/types.ts +++ b/frontend/src/components/executor/types.ts @@ -33,6 +33,13 @@ export interface ChartPriceMapping { minSpread: number; activePickField: PickSlot | null; extraLines?: ExtraLine[]; + /** + * What the chart calls each slot's line and pick hint. The slots are named + * for the grid executor ("Start", "End", "Limit"); a panel that rides them + * with different semantics — LP's upper/lower bounds — relabels them here + * rather than letting the chart mislabel its prices. + */ + lineLabels?: Partial>; } export interface ExecutorValidation { diff --git a/frontend/src/components/trade/TradeChart.tsx b/frontend/src/components/trade/TradeChart.tsx index a417ebb82..0ce2a0bc9 100644 --- a/frontend/src/components/trade/TradeChart.tsx +++ b/frontend/src/components/trade/TradeChart.tsx @@ -56,6 +56,8 @@ interface TradeChartProps { totalAmountQuote?: number; minOrderAmountQuote?: number; activePickField: PickField; + /** Per-slot overrides for the line titles and pick hints (see ChartPriceMapping). */ + lineLabels?: Partial>; onPriceSet: (field: PickSlot, price: number) => void; pricePrecision?: number; extraLines?: ExtraLine[]; @@ -86,6 +88,7 @@ export function TradeChart({ totalAmountQuote, minOrderAmountQuote, activePickField, + lineLabels, onPriceSet, pricePrecision, extraLines, @@ -637,7 +640,7 @@ export function TradeChart({ lineWidth: 2, lineStyle: mod.LineStyle.Solid, axisLabelVisible: true, - title: "Start", + title: lineLabels?.start ?? "Start", }); } @@ -648,7 +651,7 @@ export function TradeChart({ lineWidth: 2, lineStyle: mod.LineStyle.Dashed, axisLabelVisible: true, - title: "End", + title: lineLabels?.end ?? "End", }); } @@ -660,7 +663,7 @@ export function TradeChart({ lineWidth: 2, lineStyle: mod.LineStyle.Dotted, axisLabelVisible: true, - title: "Limit", + title: lineLabels?.limit ?? "Limit", }); } @@ -718,7 +721,7 @@ export function TradeChart({ extraLinesRef.current.push(pl); } } - }, [startPrice, endPrice, limitPrice, side, minSpread, totalAmountQuote, minOrderAmountQuote, activePickField, extraLines, chartReady]); + }, [startPrice, endPrice, limitPrice, side, minSpread, totalAmountQuote, minOrderAmountQuote, activePickField, extraLines, lineLabels, chartReady]); // ── Executor overlays ── // `chartReady` is a dependency, not a guard for its own sake: lightweight-charts @@ -1003,10 +1006,13 @@ export function TradeChart({ {activePickField && (

- Click on chart to set {PICK_LABELS[activePickField]} price + Click on chart to set{" "} + {(lineLabels?.[activePickField] ?? PICK_LABELS[activePickField]).toLowerCase()}{" "} + price

- Pick mode: {PICK_LABELS[activePickField]} + Pick mode:{" "} + {(lineLabels?.[activePickField] ?? PICK_LABELS[activePickField]).toLowerCase()}
)} diff --git a/frontend/src/pages/CreateExecutor.tsx b/frontend/src/pages/CreateExecutor.tsx index 9ff766734..09a0fe0c6 100644 --- a/frontend/src/pages/CreateExecutor.tsx +++ b/frontend/src/pages/CreateExecutor.tsx @@ -582,6 +582,7 @@ export function CreateExecutor() { side={chartProps.side} minSpread={chartProps.minSpread} activePickField={chartProps.activePickField} + lineLabels={chartProps.lineLabels} onPriceSet={handlePriceSet} pricePrecision={pricePrecision} extraLines={chartProps.extraLines} diff --git a/frontend/src/pages/DexPool.tsx b/frontend/src/pages/DexPool.tsx index 173568ad0..f78e0183a 100644 --- a/frontend/src/pages/DexPool.tsx +++ b/frontend/src/pages/DexPool.tsx @@ -546,6 +546,7 @@ export function DexPool() { side={active.chartProps.side} minSpread={0} activePickField={active.chartProps.activePickField} + lineLabels={active.chartProps.lineLabels} onPriceSet={active.handleChartPriceSet} extraLines={active.chartProps.extraLines} executorOverlays={overlays} From 84a253141a08f4fb727ad31c49b16cb9400ac04a Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Aug 2026 15:43:48 +0300 Subject: [PATCH 49/57] (feat) last comments from ralph --- condor/fetchers/gateway_tokens.py | 38 +++++++---- condor/preferences.py | 6 ++ condor/routine_store.py | 6 +- condor/web/routes/positions.py | 9 +++ condor/web/routes/settings.py | 7 +++ config_manager.py | 41 +++++++++--- .../src/components/trade/TradeBottomPane.tsx | 8 +++ frontend/src/components/trade/TradeChart.tsx | 6 +- frontend/src/hooks/useMainControllerData.ts | 63 ++++++++++++++++--- frontend/src/lib/api.ts | 11 ++++ tests/test_gateway_tokens.py | 28 ++++++++- 11 files changed, 189 insertions(+), 34 deletions(-) diff --git a/condor/fetchers/gateway_tokens.py b/condor/fetchers/gateway_tokens.py index 5b8af68b3..b274f9e4f 100644 --- a/condor/fetchers/gateway_tokens.py +++ b/condor/fetchers/gateway_tokens.py @@ -85,20 +85,39 @@ def token_addresses(*values: str | None) -> list[str]: return addresses +async def _network_tokens(client: Any, network_id: str) -> list[dict]: + """Gateway's whole token list for ``network_id``. + + Read whole rather than filtered because Gateway's ``search`` only looks at + symbol and name — a mint address matches nothing, and a ticker matches by + rules this module should not have to guess at. The list is a curated file + per network, so the whole of it is a small response. + """ + response = await client.gateway.get_network_tokens(network_id) + tokens = response.get("tokens", []) if isinstance(response, dict) else response + return [token for token in (tokens or []) if isinstance(token, dict)] + + async def _is_listed(client: Any, network_id: str, address: str) -> bool: """Whether Gateway's token list for ``network_id`` already holds ``address``. - Searched rather than listed in full: Gateway matches ``search`` against symbol, - name *and* address, so this is one small response instead of the whole list. - An API that narrowed the search to symbols would only cost a redundant save, - which is itself a no-op — so a false negative here is safe. + The whole list is read, not ``search=
``: Gateway matches ``search`` + against symbol and name only, so asking for a mint always answered zero + tokens. Every token then looked unlisted, the save that followed was refused + with "already exists" (a *symbol* check, which the listed token's own ticker + trips), and the re-read said unlisted again — so a correctly listed token + like Solana USDC reported ``symbol_taken`` forever, the pool banner asked the + user to add a token that was already there, and the conflict lookup could + name no holder but the token itself. + + A network's list is Gateway's curated file — tens of entries, not the whole + chain — so reading it whole is one small response, and confirmed hits are + memoized by the caller anyway. """ - response = await client.gateway.get_network_tokens(network_id, search=address) - tokens = response.get("tokens", []) if isinstance(response, dict) else response wanted = address.lower() return any( str((token or {}).get("address") or "").lower() == wanted - for token in (tokens or []) + for token in await _network_tokens(client, network_id) ) @@ -113,12 +132,9 @@ async def find_symbol_holder( being registered itself, so a half-written state cannot name it as its own conflict. """ - response = await client.gateway.get_network_tokens(network_id, search=symbol) - tokens = response.get("tokens", []) if isinstance(response, dict) else response wanted = symbol.lower() excluded = (exclude or "").lower() - for token in tokens or []: - entry = token or {} + for entry in await _network_tokens(client, network_id): if str(entry.get("symbol") or "").lower() != wanted: continue address = str(entry.get("address") or "") diff --git a/condor/preferences.py b/condor/preferences.py index 014fe2c10..f11cb425c 100644 --- a/condor/preferences.py +++ b/condor/preferences.py @@ -92,6 +92,12 @@ def _load_from_cm(user_data: Dict) -> None: USER_PREFERENCES_KEY = "user_preferences" _MIGRATION_DONE_KEY = "_prefs_migration_done" +# Marks a ``user_data`` built *for* one server — a routine or agent run launched +# against it — rather than a chat's own preferences. ``get_effective_server`` +# honours a pinned server outright; without the mark it treats ``active_server`` +# as a cache of the chat default and lets config.yml correct it. +SERVER_PIN_KEY = "_server_pinned" + # Portfolio defaults DEFAULT_PORTFOLIO_DAYS = 3 DEFAULT_PORTFOLIO_INTERVAL = "1h" diff --git a/condor/routine_store.py b/condor/routine_store.py index 23aeb2cdf..e35cb39e8 100644 --- a/condor/routine_store.py +++ b/condor/routine_store.py @@ -134,7 +134,7 @@ class WebRoutineContext: """Lightweight context so routines can run without Telegram.""" def __init__(self, server_name: str, bot=None, chat_id: int = 0): - from condor.preferences import USER_PREFERENCES_KEY + from condor.preferences import SERVER_PIN_KEY, USER_PREFERENCES_KEY self._chat_id = chat_id # The server this run was launched against, readable by name — for a routine @@ -146,8 +146,12 @@ def __init__(self, server_name: str, bot=None, chat_id: int = 0): # "preferences" while the preference API reads USER_PREFERENCES_KEY # ("user_preferences"), so the server a web or agent run was launched with # never reached its client — every such run silently picked the chat default. + # Pinned, not preferred: this run was launched against `server_name`, so + # the chat's own default must not override it — the chat may well be a + # chat whose default is a different server (see `get_effective_server`). self._user_data: dict[str, Any] = { USER_PREFERENCES_KEY: {"general": {"active_server": server_name}}, + SERVER_PIN_KEY: True, } @property diff --git a/condor/web/routes/positions.py b/condor/web/routes/positions.py index d7baee46a..e6d00c09c 100644 --- a/condor/web/routes/positions.py +++ b/condor/web/routes/positions.py @@ -22,6 +22,14 @@ def _normalize_position(pos: dict, source: str, source_name: str) -> dict: notional_value = abs(float(amount)) * float(entry_price) except (ValueError, TypeError): notional_value = 0 + # Which executors this row aggregates. The only attribution the API gives a + # held position: a position has no pool of its own, but an LP executor + # records the pool it opened in, so the pool workspace can tell a hold that + # belongs to the pool on screen from one held in another pool of the same + # pair on the same chain. + executor_ids = pos.get("executor_ids") or [] + if not isinstance(executor_ids, list): + executor_ids = [] return { "connector_name": pos.get("connector_name") or pos.get("connector") or "", "trading_pair": pos.get("trading_pair") or "", @@ -38,6 +46,7 @@ def _normalize_position(pos: dict, source: str, source_name: str) -> dict: "realized_pnl": pos.get("realized_pnl_quote") or 0, "cum_fees": pos.get("cum_fees_quote") or 0, "executor_count": pos.get("executor_count") or 0, + "executor_ids": [str(i) for i in executor_ids], "source": source, "source_name": source_name, } diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index 66fc57cc4..df1bb2411 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -166,6 +166,13 @@ async def set_default_server(name: str, user: WebUser = Depends(get_current_user if not cm.has_server_access(user.id, name): raise HTTPException(status_code=404, detail="Server not found") cm.set_chat_default_server(user.id, name) + # Both records Telegram's own "set default" writes, or the two disagree: + # `chat_defaults` is what `get_effective_server` resolves, while the general + # preference is what a routine, an agent or MCP reads as the active server. + # Writing only the first left those surfaces pinned to the previous choice. + from condor.preferences import load_user_data_for, set_active_server + + set_active_server(load_user_data_for(user.id), name) return {"default": True, "name": name} diff --git a/config_manager.py b/config_manager.py index 38b0638c1..358d30924 100644 --- a/config_manager.py +++ b/config_manager.py @@ -1081,10 +1081,20 @@ def get_effective_server(chat_id: int, user_data: dict = None) -> str | None: """Get the effective default server for a chat, checking both user_data and config.yml. Priority: - 1. user_data active_server (from pickle, fast in-memory) - 2. chat_defaults from config.yml (persistent across hard kills) + 1. chat_defaults from config.yml — the one place *every* surface writes when + the user picks a default (Telegram's /servers, the web dashboard, admin + assignment), so it is the only entry that can be current. + 2. user_data active_server (from pickle, per-user, covers chats that never + had a default of their own) 3. None if nothing configured + The order used to be the other way round, and that made the web dashboard's + "set default" invisible to a live Telegram session: the web writes + config.yml, while the pickle in the running bot's ``user_data`` kept the + server chosen before it and won every lookup until the process restarted. + The pickle is a cache of the choice, not the choice — so it is read second + and re-synced from config.yml whenever the two disagree. + Args: chat_id: The chat ID user_data: Optional user_data dict from context @@ -1092,25 +1102,36 @@ def get_effective_server(chat_id: int, user_data: dict = None) -> str | None: Returns: Server name or None """ + from condor.preferences import SERVER_PIN_KEY from handlers.config.user_preferences import get_active_server - # First check user_data (pickle - might be lost on hard kill) - if user_data: - active = get_active_server(user_data) - if active: - return active + # A context built for one server — a routine or an agent run launched + # against it — carries its own answer and is never re-resolved from a chat's + # ambient default. Everything else reaching this point holds a preference, + # which is a cache of a choice config.yml records authoritatively. + if user_data and user_data.get(SERVER_PIN_KEY): + pinned = get_active_server(user_data) + if pinned: + return pinned - # Fall back to chat_defaults in config.yml (always persisted) cm = get_config_manager() chat_default = cm._data.get("chat_defaults", {}).get(chat_id) if chat_default and chat_default in cm._data.get("servers", {}): - # Also sync back to user_data for fast future access - if user_data is not None: + # Keep the pickle in step, but only on an actual change: the setter + # persists the whole preference section to config.yml, and this runs on + # every server lookup. + if user_data is not None and get_active_server(user_data) != chat_default: from handlers.config.user_preferences import set_active_server set_active_server(user_data, chat_default) return chat_default + # No default recorded for this chat — fall back to the user's own last pick. + if user_data: + active = get_active_server(user_data) + if active: + return active + return None diff --git a/frontend/src/components/trade/TradeBottomPane.tsx b/frontend/src/components/trade/TradeBottomPane.tsx index 2a1bae44e..10702608e 100644 --- a/frontend/src/components/trade/TradeBottomPane.tsx +++ b/frontend/src/components/trade/TradeBottomPane.tsx @@ -626,6 +626,14 @@ export function TradeBottomPane({ }`}> {pos.position_side?.toUpperCase() ?? "—"} + {pos.pool_scoped === false && ( + + any pool + + )}
{Math.abs(pos.amount).toFixed(4)} diff --git a/frontend/src/components/trade/TradeChart.tsx b/frontend/src/components/trade/TradeChart.tsx index 0ce2a0bc9..8bac28847 100644 --- a/frontend/src/components/trade/TradeChart.tsx +++ b/frontend/src/components/trade/TradeChart.tsx @@ -949,7 +949,11 @@ export function TradeChart({ const pnlStr = _cvtPnl2 ? _cvtPnl2(pnl) : (Math.abs(pnl) >= 1000 ? `${pnl >= 0 ? "+" : ""}$${(pnl / 1000).toFixed(1)}K` : `${pnl >= 0 ? "+" : ""}$${pnl.toFixed(2)}`); const amt = Math.abs(pos.amount); const color = pnlHexColor(pnl); - const label = `${isLong ? "LONG" : "SHORT"} ${amt.toFixed(4)} · ${pnlStr}`; + // A hold nothing ties to this pool still belongs on the chart -- the price + // is the same market -- but it must not read as this pool's position, so + // it says whose it is: the pair's, across whatever pool the router used. + const scope = pos.pool_scoped === false ? " · any pool" : ""; + const label = `${isLong ? "LONG" : "SHORT"} ${amt.toFixed(4)} · ${pnlStr}${scope}`; const pl = series.createPriceLine({ price: pos.entry_price, color, diff --git a/frontend/src/hooks/useMainControllerData.ts b/frontend/src/hooks/useMainControllerData.ts index d2dbc5467..fee0375a6 100644 --- a/frontend/src/hooks/useMainControllerData.ts +++ b/frontend/src/hooks/useMainControllerData.ts @@ -149,6 +149,26 @@ export function useMainControllerData( placeholderData: keepPreviousData, // keep showing old data during refetch/refresh }); + /** + * Executor id -> the pool it traded in, for every row we hold. + * + * Built from the *unfiltered* queries on purpose: the pool filter above drops + * a foreign pool's LP executor, and that row is exactly the evidence needed to + * recognise a held position as belonging somewhere else. + */ + const executorPools = useMemo(() => { + const byId = new Map(); + for (const ex of [ + ...(cachedExecutors ?? []), + ...(altExecutors ?? []), + ...(poolLpExecutors ?? []), + ]) { + const pool = executorPool(ex); + if (pool) byId.set(ex.id, pool); + } + return byId; + }, [cachedExecutors, altExecutors, poolLpExecutors]); + const positions = useMemo(() => { if (!positionsData) return []; const all = [ @@ -163,14 +183,41 @@ export function useMainControllerData( ...(wantsAlt ? [altPair] : []), ...executors.map((ex) => ex.trading_pair).filter(Boolean), ]); - return all.filter( - (p) => - // Show positions from main controller or untagged (executor-level positions) - (!p.controller_id || p.controller_id === "main") && - p.connector_name === connector && - pairs.has(p.trading_pair), - ); - }, [positionsData, connector, pair, altPair, wantsAlt, executors]); + return all + .filter( + (p) => + // Show positions from main controller or untagged (executor-level positions) + (!p.controller_id || p.controller_id === "main") && + p.connector_name === connector && + pairs.has(p.trading_pair), + ) + .map((p) => { + // On a DEX pool page the connector is the *network*, so connector+pair + // alone puts one SOL-USDC hold on every Solana SOL-USDC pool -- Orca's + // and Meteora's alike -- and each page reads as if the position were + // its own. The executors behind the row are the only attribution there + // is: an LP executor names its pool, and a swap names none because the + // aggregator picked the route. + if (!poolAddress) return p; + const pools = (p.executor_ids ?? []) + .map((id) => executorPools.get(id)) + .filter((pool): pool is string => !!pool); + if (!pools.length) return { ...p, pool_scoped: false }; + return pools.includes(poolAddress) + ? { ...p, pool_scoped: true } + : null; + }) + .filter((p): p is NonNullable => p !== null); + }, [ + positionsData, + connector, + pair, + altPair, + wantsAlt, + executors, + poolAddress, + executorPools, + ]); return { executors, overlays, positions, isLoadingPositions }; } diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 85f3d2613..90f3a3936 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -232,10 +232,21 @@ export interface ConsolidatedPosition { realized_pnl: number; cum_fees: number; executor_count: number; + /** The executors this row aggregates — how a hold is traced back to a pool. */ + executor_ids?: string[]; leverage: number; controller_id: string; source: "executor" | "bot"; source_name: string; + /** + * Set only on a pool workspace: whether this hold is known to be *this* pool's. + * + * `true` when a contributing LP executor records the pool on screen, `false` + * when nothing ties it to any pool — a swap is routed by the aggregator, so + * the position it leaves behind is a fact about the pair, not about a pool. + * Undefined off a pool page, where the question does not arise. + */ + pool_scoped?: boolean; } export interface ConsolidatedPositionsResponse { diff --git a/tests/test_gateway_tokens.py b/tests/test_gateway_tokens.py index c525bece3..be59a8dc0 100644 --- a/tests/test_gateway_tokens.py +++ b/tests/test_gateway_tokens.py @@ -44,15 +44,17 @@ async def get_network_tokens(self, network_id, search=None): if self.fail_search: raise RuntimeError("gateway down") self.searches.append(search) - # Gateway matches search against symbol, name and address alike. + # Gateway's `search` matches symbol and name only — never an address. + # Modelling it any other way is what hid the bug this fake now pins: + # a listed token searched for by mint came back empty and read unlisted. needle = (search or "").lower() return { "tokens": [ t for t in self.tokens if not needle - or needle in str(t["address"]).lower() or needle in str(t.get("symbol") or "").lower() + or needle in str(t.get("name") or "").lower() ] } @@ -105,7 +107,27 @@ def test_a_known_token_is_never_saved_twice(): assert second == {MINT: "listed"} # One save, and no second lookup: the memo answers the repeat visit outright. assert gw.saves == [MINT] - assert gw.searches.count(MINT) == 2 # the pre-check and the verification + # The pre-check and the post-save verification, both reading the list whole + # (`search=None`) because Gateway cannot look a mint up by address. + assert gw.searches == [None, None] + + +def test_a_listed_token_is_not_asked_to_be_added_again(): + """The bug the pool banner showed: Solana USDC, listed, reported unlisted. + + Gateway's token search ignores addresses, so a lookup by mint answers + nothing. Reading that as "not listed" made the save that followed collide + with the token's *own* ticker ("already exists"), the verdict come back + `symbol_taken`, and the banner ask the user to add a token that was already + on the list — with no holder to name and no Replace that could help. + """ + gw = FakeGateway() + gw.tokens = [{"address": USDC, "symbol": "USDC", "name": "USDC"}] + + verdicts = asyncio.run(ensure_tokens_listed(FakeClient(gw), NETWORK, [USDC])) + + assert verdicts == {USDC: "listed"} + assert gw.saves == [] def test_a_save_that_wrote_nothing_is_reported_not_assumed(): From bdf07d9a0fcd0c77aadd39a2c3751b912a89e3ae Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Aug 2026 19:02:00 +0300 Subject: [PATCH 50/57] (feat) fix portfolio --- utils/auth.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/utils/auth.py b/utils/auth.py index cdb7ddb37..bcc47e966 100644 --- a/utils/auth.py +++ b/utils/auth.py @@ -36,6 +36,35 @@ async def _notify_admin_new_user( logger.warning(f"Failed to notify admin about new user: {e}") +def _refresh_active_server(update: Update, context: ContextTypes.DEFAULT_TYPE) -> None: + """Re-sync the cached ``active_server`` from config.yml before the handler runs. + + ``chat_defaults`` in config.yml is where every surface records the chosen + default, but most handlers read the ``user_data`` copy of it + (``get_active_server``) rather than resolving through + ``get_effective_server``. That copy is only rewritten when something happens + to resolve — so a default picked on the web dashboard stayed invisible to + /portfolio, /bots and the gateway screens until the user opened /servers and + incidentally triggered a resolve. + + Running the resolve here, at the one gate every Telegram update passes + through, keeps the copy true for whatever the handler does with it. It is a + dict lookup on the already-loaded config in the common case; the write only + happens on an actual change. + """ + if context.user_data is None: + return + chat = getattr(update, "effective_chat", None) + if chat is None: + return + try: + from config_manager import get_effective_server + + get_effective_server(chat.id, context.user_data) + except Exception as e: # never block a command over a preference refresh + logger.debug("Could not refresh active server: %s", e) + + def restricted(func): """ Decorator that checks if user is approved. @@ -98,6 +127,7 @@ async def wrapped( # User is approved (USER or ADMIN role) # Store user_id in context for access control in subsequent calls context.user_data["_user_id"] = user_id + _refresh_active_server(update, context) return await func(update, context, *args, **kwargs) return wrapped From 43eab080acf1dc5025e05d7ad97ec25b0f436699 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Aug 2026 19:51:19 +0300 Subject: [PATCH 51/57] (feat) task notifications reach the dashboard: per-user store, WS push and the four producers Everything Condor said when a background task finished went to Telegram and nowhere else, so a user with only the dashboard open learned nothing. condor/notifications.py is the other half: a per-user JSON store under data/ (atomic, capped, newest first, serialised behind a write lock) plus a push bus that surfaces register themselves on, the same shape runtime/wake.py already uses. chat_ws registers a sink emitting a `notification` event on the socket every authenticated page already holds; GET/POST /api/v1/notifications is the durable path for a notice raised while no tab was open. The four producers -- a finished delegation, a finished routine, /agents/notify and the boot notice -- record alongside their existing Telegram push, with the same text, so the surfaces cannot tell different stories about one task. A routine with no conversation behind it (the scheduler, the dashboard) now reaches its owner for the first time. NotifyBot is the last rung of resolve_bot(): with no TELEGRAM_TOKEN at all an outbound message becomes a bell item instead of being dropped by a sender that cannot deliver. It sits below both Telegram rungs, so a configured install is unchanged and nothing is ever pushed to Telegram twice. FEAT-048 --- condor/agents/delegate.py | 36 ++- condor/notifications.py | 317 ++++++++++++++++++++++ condor/routine_store.py | 36 ++- condor/web/app.py | 2 + condor/web/routes/agents.py | 13 + condor/web/routes/chat_ws.py | 30 +- condor/web/routes/notifications.py | 48 ++++ main.py | 22 +- tests/conftest.py | 14 + tests/test_notifications.py | 421 +++++++++++++++++++++++++++++ tests/test_notify_route.py | 9 +- 11 files changed, 932 insertions(+), 16 deletions(-) create mode 100644 condor/notifications.py create mode 100644 condor/web/routes/notifications.py create mode 100644 tests/test_notifications.py diff --git a/condor/agents/delegate.py b/condor/agents/delegate.py index 618e90245..e43292b8c 100644 --- a/condor/agents/delegate.py +++ b/condor/agents/delegate.py @@ -689,11 +689,14 @@ def resolve_bot(bot=None): """The best Telegram sender available in this process. Prefer the passed live ``bot``; otherwise fall back to the registered routine - bot, and finally the ``_HttpBot`` Telegram-HTTP path (``TELEGRAM_TOKEN``) that + bot, then the ``_HttpBot`` Telegram-HTTP path (``TELEGRAM_TOKEN``) that routines/notification already use, so a process with no live bot still - delivers. Public because a Telegram wake sink (FEAT-034) must reach the user - through exactly the same ladder a completion notice does -- two ladders would - eventually disagree about who can talk to a chat. + delivers. With no token at all the last rung is ``NotifyBot`` (FEAT-048), + which records the message as a dashboard notification -- an install with no + Telegram now tells the user things instead of dropping them. Public because a + Telegram wake sink (FEAT-034) must reach the user through exactly the same + ladder a completion notice does -- two ladders would eventually disagree + about who can talk to a chat. """ target = bot if target is None: @@ -706,7 +709,16 @@ def resolve_bot(bot=None): if target is None: from condor.routine_store import _HttpBot - target = _HttpBot() + http = _HttpBot() + if http.has_token(): + return http + # No Telegram at all: record on the dashboard instead of handing the + # message to a sender that drops it (FEAT-048). This rung sits *below* + # both Telegram rungs, so an install with a bot is byte-for-byte + # unchanged and nothing is ever delivered twice to Telegram. + from condor.notifications import NotifyBot + + return NotifyBot() return target @@ -715,4 +727,16 @@ async def _notify_done(dt: DelegateTask, bot) -> None: if not dt.chat_id: return - await resolve_bot(bot).send_message(chat_id=dt.chat_id, text=_completion_text(dt)) + from condor.notifications import NotifyBot, record + + text = _completion_text(dt) + target = resolve_bot(bot) + await target.send_message(chat_id=dt.chat_id, text=text) + + # And on the dashboard bell (FEAT-048), with the same text the Telegram push + # and the transcript note carry, so the three surfaces cannot tell three + # stories about one task. Skipped when the resolved sender *is* the bell: + # that rung only wins when there is no Telegram, and it has already recorded + # this very message. + if not isinstance(target, NotifyBot): + await record(dt.user_id, text, kind="delegation") diff --git a/condor/notifications.py b/condor/notifications.py new file mode 100644 index 000000000..a30506153 --- /dev/null +++ b/condor/notifications.py @@ -0,0 +1,317 @@ +"""Per-user task notifications, and the surfaces they reach (FEAT-048). + +Everything Condor tells a user when a background task finishes — a delegation +completing, a routine run ending, an agent's ``send_notification``, the boot +notice — used to be pushed to Telegram and nowhere else, so a user who only had +the dashboard open learned nothing. This module is the other half: a small +per-user store plus a push bus, so a finished task lights the dashboard's bell +live *and* is still there, unread, an hour later. + +Two rules shape it: + +* **The notice must survive nobody watching.** So a store, not just a push: + ``data/notifications.json``, newest first, capped per user, written with + :func:`condor.fsutil.atomic_write_json` exactly like ``routine_hooks.json``. +* **Core must not import the web layer.** The surface registers itself at + import time, the same shape ``condor/runtime/wake.py`` uses for wake and note + sinks — see ``condor/web/routes/chat_ws.py``. + +:class:`NotifyBot` is the adapter that gets this to every *existing* producer +without touching their call sites: it implements the four methods the codebase's +bot surface actually uses and records instead of sending, so it can sit at the +bottom of ``resolve_bot()``'s ladder and catch anything that has no Telegram to +go to. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +import time +import uuid +from dataclasses import asdict, dataclass +from pathlib import Path +from typing import Any, Awaitable, Callable + +from condor.fsutil import atomic_write_json + +log = logging.getLogger(__name__) + +_FILE = Path("data") / "notifications.json" + +# Per user. A bell shows the last few dozen; anything older is history nobody +# scrolls to, and an unbounded file would be read and rewritten on every record. +_MAX_PER_USER = 100 + +# Serialises the read-modify-write in :func:`record`. Two producers finishing at +# once would otherwise each read the same file and the second write would drop +# the first's notification. The write rate is a handful per hour, so the lock +# costs nothing and removes the whole class of lost updates. +_write_lock = asyncio.Lock() + + +@dataclass(frozen=True) +class Notification: + """One thing worth telling a user about, addressed to the user.""" + + id: str + user_id: int + ts: float + kind: str # "delegation" | "routine" | "agent" | "system" + text: str + title: str | None = None + # A dashboard route, e.g. "/routines?tab=reports". Relative on purpose: the + # bell navigates within the SPA, it does not open the world. + link: str | None = None + read: bool = False + + def to_wire(self) -> dict[str, Any]: + return asdict(self) + + +# ── The push bus ── + +PushSink = Callable[[Notification], Awaitable[None]] + +_push_sinks: list[PushSink] = [] + + +def register_push_sink(sink: PushSink) -> None: + """Bind a live surface's renderer. Called by the surface, at import time.""" + if sink not in _push_sinks: + _push_sinks.append(sink) + + +async def _push(notification: Notification) -> None: + """Fan out to whoever is watching. A dead sink is not a dead notification. + + Mirrors ``wake._guard``: the store is the record, so a socket that blew up + must never propagate into the producer that was only announcing something. + """ + for sink in list(_push_sinks): + try: + await sink(notification) + except Exception: # noqa: BLE001 - a dead sink is not a dead notification + log.warning("Notification push sink failed", exc_info=True) + + +# ── The store ── + + +def _read_all() -> dict[str, list[dict]]: + if not _FILE.exists(): + return {} + try: + data = json.loads(_FILE.read_text(encoding="utf-8")) + if not isinstance(data, dict): + return {} + return {str(k): v for k, v in data.items() if isinstance(v, list)} + except Exception: # noqa: BLE001 + log.warning("Failed to read %s; treating as empty", _FILE) + return {} + + +def _write_all(data: dict[str, list[dict]]) -> None: + atomic_write_json(_FILE, data, indent=2) + + +def _hydrate(raw: dict) -> Notification | None: + """One stored dict back into a :class:`Notification`, or None if it is junk. + + Tolerant on purpose: a file written by an older build (or by hand) must not + take the whole bell down with it. + """ + try: + return Notification( + id=str(raw["id"]), + user_id=int(raw["user_id"]), + ts=float(raw.get("ts") or 0.0), + kind=str(raw.get("kind") or "system"), + text=str(raw.get("text") or ""), + title=raw.get("title") or None, + link=raw.get("link") or None, + read=bool(raw.get("read")), + ) + except (KeyError, TypeError, ValueError): + return None + + +async def record( + user_id: int | None, + text: str, + *, + kind: str = "system", + title: str | None = None, + link: str | None = None, +) -> Notification | None: + """Store a notification for ``user_id`` and push it to live surfaces. + + Returns the stored notification, or ``None`` when there is nobody to + address it to (a falsy ``user_id``) or nothing to say — a producer with no + user behind it must not be an error, it simply has no bell to ring. + """ + if not user_id or not text: + return None + + notification = Notification( + id=uuid.uuid4().hex, + user_id=int(user_id), + ts=time.time(), + kind=kind, + text=text, + title=title, + link=link, + ) + + async with _write_lock: + data = _read_all() + key = str(notification.user_id) + items = [notification.to_wire()] + (data.get(key) or []) + data[key] = items[:_MAX_PER_USER] + try: + _write_all(data) + except Exception: # noqa: BLE001 - an unwritable store still gets a push + log.warning("Could not persist notification for %s", user_id, exc_info=True) + + await _push(notification) + return notification + + +def list_for(user_id: int, limit: int = 50) -> list[Notification]: + """This user's notifications, newest first. Never anyone else's.""" + if not user_id: + return [] + raw = _read_all().get(str(int(user_id))) or [] + items = [n for n in (_hydrate(r) for r in raw if isinstance(r, dict)) if n] + return items[: max(0, limit)] + + +def unread_count(user_id: int) -> int: + """How many of this user's notifications are still unread.""" + return sum(1 for n in list_for(user_id, limit=_MAX_PER_USER) if not n.read) + + +def mark_read(user_id: int, ids: list[str] | None = None) -> int: + """Mark ``ids`` read (``None`` = all of them). Returns the unread count left. + + Scoped to ``user_id``'s own bucket, so an id belonging to someone else is + simply not found — there is no way to reach across users from here. + """ + if not user_id: + return 0 + key = str(int(user_id)) + data = _read_all() + items = data.get(key) or [] + wanted = set(ids) if ids is not None else None + changed = False + for raw in items: + if not isinstance(raw, dict): + continue + if wanted is not None and str(raw.get("id")) not in wanted: + continue + if not raw.get("read"): + raw["read"] = True + changed = True + if changed: + data[key] = items + try: + _write_all(data) + except Exception: # noqa: BLE001 + log.warning("Could not persist read state for %s", user_id, exc_info=True) + return sum(1 for raw in items if isinstance(raw, dict) and not raw.get("read")) + + +def user_for_chat(chat_id: Any) -> int | None: + """The dashboard user behind a Telegram chat id, or None if there is none. + + A private chat's id *is* the user id — the rule ``_check_chat_access`` + already relies on. A group chat id (negative) has no dashboard owner, and + neither does a private chat with someone this install has never seen, so + both return None: guessing would file one person's message in another + person's bell. + """ + try: + user_id = int(chat_id) + except (TypeError, ValueError): + return None + if user_id <= 0: # Telegram group/channel ids are negative. + return None + try: + from config_manager import get_config_manager + + if get_config_manager().get_user(user_id) is None: + log.debug("No dashboard user for chat %s; not recording", chat_id) + return None + except Exception: # noqa: BLE001 - config unavailable: do not guess + log.debug("Could not resolve chat %s to a user", chat_id, exc_info=True) + return None + return user_id + + +# ── The adapter every existing producer already talks to ── + + +class NotifyBot: + """A bot-shaped object that records notifications instead of sending them. + + The four methods are the ones the codebase's bot surface actually uses — + the same set ``_HttpBot`` implements (``condor/routine_store.py``). Sitting + at the bottom of ``resolve_bot()``'s ladder, this is what makes a routine's + ``context.bot.send_message(...)`` land on the dashboard in an install with + no Telegram token, with no routine and no call-site changes. + + Addressing goes through :func:`user_for_chat`, so nothing is recorded for a + chat id that is not a known user. + + Binary payloads are **not** re-hosted. ``send_photo``/``send_document`` + record the caption (or the filename) and link to the reports browser, which + already serves routine reports through ``GET /reports/{id}/html``. + """ + + def __init__(self, kind: str = "system"): + self._kind = kind + + # -- helpers -- + + @staticmethod + def _arg(a: tuple, kw: dict, name: str, index: int) -> Any: + return kw.get(name) if name in kw else (a[index] if len(a) > index else None) + + async def _record(self, chat_id: Any, text: str, link: str | None = None): + user_id = user_for_chat(chat_id) + if user_id is None: + return None + return await record(user_id, text, kind=self._kind, link=link) + + # -- the bot surface -- + + async def send_message(self, *a, **kw): + chat_id = self._arg(a, kw, "chat_id", 0) + text = self._arg(a, kw, "text", 1) or "" + return await self._record(chat_id, text) + + async def send_photo(self, *a, **kw): + chat_id = self._arg(a, kw, "chat_id", 0) + text = kw.get("caption") or "Chart" + return await self._record(chat_id, text, link="/routines?tab=reports") + + async def send_document(self, *a, **kw): + chat_id = self._arg(a, kw, "chat_id", 0) + document = self._arg(a, kw, "document", 1) + text = ( + kw.get("caption") + or kw.get("filename") + or getattr(document, "name", None) + or "Report" + ) + return await self._record(chat_id, text, link="/routines?tab=reports") + + async def edit_message_text(self, *a, **kw): + """A live-updating message has no bell equivalent: nothing to record. + + The bell is a log of things that happened, not a surface that redraws — + recording every tick of a ``LiveReport`` would bury the notices that + matter under a routine's own progress updates. + """ + return None diff --git a/condor/routine_store.py b/condor/routine_store.py index e35cb39e8..5a5433bea 100644 --- a/condor/routine_store.py +++ b/condor/routine_store.py @@ -94,6 +94,14 @@ async def edit_message_text(self, *a, **kw): data = {k: v for k, v in kw.items() if v is not None} return await self._post("editMessageText", data) + def has_token(self) -> bool: + """Can this actually deliver? Without a token every send is a no-op. + + Read by ``delegate.resolve_bot`` so the ladder can fall through to the + dashboard bell rather than to a sender that silently drops the message. + """ + return bool(self._token) + async def get_chat_member(self, *a, **kw): """Raw ``getChatMember`` envelope — the web routes' chat-ownership check (SEC-198) reads ``result.status`` out of it when no live bot is around.""" @@ -441,17 +449,35 @@ async def _report_run( worth showing, not worth a model turn to announce it. A run with no conversation behind it — the scheduler, the dashboard, the - Telegram menu, an instance restored on boot — is a no-op: ``record_system`` - ignores an empty id and a run with no session key reaches no surface. - Neither delivery is allowed to raise: a missing note must not cost the - user the run's own result or its hooks. + Telegram menu, an instance restored on boot — still lights the owner's + bell (FEAT-048) and is otherwise a no-op: ``record_system`` ignores an + empty id and a run with no session key reaches no surface. No delivery + is allowed to raise: a missing note must not cost the user the run's own + result or its hooks. """ meta = self._instances.get(instance_id) or {} + text = _run_outcome_text(meta.get("routine_name") or "", summary, error) + + # The bell is addressed to the *owner*, not to a conversation, so it is + # the one surface a scheduled or dashboard-started run can reach. + try: + from condor.notifications import record + + await record( + meta.get("user_id"), + text, + kind="routine", + link="/routines?tab=reports", + ) + except Exception: + logger.debug( + f"Could not notify owner about run {instance_id}", exc_info=True + ) + conversation_id = meta.get("conversation_id") or "" if not conversation_id: return - text = _run_outcome_text(meta.get("routine_name") or "", summary, error) try: from condor.runtime.conversations import record_system diff --git a/condor/web/app.py b/condor/web/app.py index e5ca6d623..18e7ac323 100644 --- a/condor/web/app.py +++ b/condor/web/app.py @@ -26,6 +26,7 @@ executors, market, meta, + notifications, portfolio, positions, reports, @@ -87,6 +88,7 @@ def create_app() -> FastAPI: app.include_router(market.router, prefix="/api/v1") app.include_router(dex.router, prefix="/api/v1") app.include_router(meta.router, prefix="/api/v1") + app.include_router(notifications.router, prefix="/api/v1") app.include_router(ws.router, prefix="/api/v1") app.include_router(agents.router, prefix="/api/v1") app.include_router(routines.router, prefix="/api/v1") diff --git a/condor/web/routes/agents.py b/condor/web/routes/agents.py index a6faa02c1..5ac8daa26 100644 --- a/condor/web/routes/agents.py +++ b/condor/web/routes/agents.py @@ -1275,6 +1275,19 @@ async def notify_user(req: NotifyRequest, user: WebUser = Depends(get_current_us exc_info=True, ) + # The dashboard bell (FEAT-048), addressed to the caller themselves — never + # to ``req.chat_id``, which may legitimately be a group they belong to but + # which has no dashboard owner. This is what makes ``send_notification`` + # succeed on an install with no Telegram: the tool already counts + # ``recorded`` as delivered. + try: + from condor.notifications import record + + if await record(user.id, req.text, kind="agent"): + recorded = True + except Exception: + log.debug("Could not record a notification for user %s", user.id, exc_info=True) + sent = False if req.chat_id: sent = await _push_to_telegram(req.chat_id, req.text, req.parse_mode) diff --git a/condor/web/routes/chat_ws.py b/condor/web/routes/chat_ws.py index 0652c0a1b..2f9719735 100644 --- a/condor/web/routes/chat_ws.py +++ b/condor/web/routes/chat_ws.py @@ -15,6 +15,7 @@ from condor.llm.openrouter_models import fetch_models from condor.llm.options import DEFAULT_AGENT +from condor.notifications import Notification, register_push_sink from condor.runtime import WEB, EventType, PromptRequest, SessionKey, SessionSpec from condor.runtime import client as runtime from condor.runtime import conversations @@ -295,10 +296,37 @@ async def _deliver_note( await _send(ws, message) +async def _push_notification(notification: Notification) -> None: + """Light this user's bell in every tab they have open (FEAT-048). + + A third out-of-band push on this socket, next to ``permission_request`` and + ``system_note``. Unlike those two it is addressed to the *user*, not to a + conversation — a finished background task has no slot — so the frame + carries no ``slot_id`` and the store, not this push, is what guarantees the + notice is seen: a user with no tab open simply has no entry here, and the + bell picks it up from ``GET /notifications`` on the next load. + """ + for ws in list(_attached_sockets.get(notification.user_id, ())): + await _send( + ws, + { + "event": "notification", + "id": notification.id, + "kind": notification.kind, + "title": notification.title, + "text": notification.text, + "link": notification.link, + "ts": notification.ts, + }, + ) + + # Registered here rather than imported by the runtime: ``condor.runtime`` must -# not depend on web or handler code (see ``client._local()``). +# not depend on web or handler code (see ``client._local()``). ``condor.notifications`` +# is registered the same way and for the same reason. register_sink_factory(WEB, _wake_sink) register_note_sink(WEB, _deliver_note) +register_push_sink(_push_notification) @router.websocket("/ws/chat") diff --git a/condor/web/routes/notifications.py b/condor/web/routes/notifications.py new file mode 100644 index 000000000..61e5d7ee6 --- /dev/null +++ b/condor/web/routes/notifications.py @@ -0,0 +1,48 @@ +"""Task notifications as an addressable resource (FEAT-048). + +The WS ``notification`` event (``chat_ws.py``) is the low-latency path; this is +the durable one, so a notice raised while no tab was open is still in the bell +on the next page load. + +Everything here is scoped to ``user.id`` from the JWT and never to a +body-supplied id — a notification is addressed to a person, and the store is the +only thing standing between one person's finished tasks and another's. +""" + +from __future__ import annotations + +from fastapi import APIRouter, Depends, Query +from pydantic import BaseModel + +from condor import notifications +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +router = APIRouter(prefix="/notifications", tags=["notifications"]) + + +class MarkReadRequest(BaseModel): + """Which notifications to mark read. ``None`` (the default) means all.""" + + ids: list[str] | None = None + + +@router.get("") +async def list_notifications( + limit: int = Query(default=50, ge=1, le=100), + user: WebUser = Depends(get_current_user), +): + """This user's notifications, newest first, with the unread count.""" + items = notifications.list_for(user.id, limit=limit) + return { + "items": [n.to_wire() for n in items], + "unread": notifications.unread_count(user.id), + } + + +@router.post("/read") +async def mark_notifications_read( + body: MarkReadRequest, user: WebUser = Depends(get_current_user) +): + """Mark some (or all) of this user's notifications read.""" + return {"unread": notifications.mark_read(user.id, body.ids)} diff --git a/main.py b/main.py index db1b7c4c4..7522a4f78 100644 --- a/main.py +++ b/main.py @@ -534,10 +534,22 @@ async def _notify_interrupted_runs(bot, report) -> None: for run in runs: suffix = " — restarted" if run.restarted else "" lines.append(f"• {run.label} (last tick {run.last_tick}){suffix}") + text = "\n".join(lines) try: - await bot.send_message(chat_id=chat_id, text="\n".join(lines)) + await bot.send_message(chat_id=chat_id, text=text) except Exception: logger.warning("Could not notify chat %s about interrupted runs", chat_id) + # And on the bell (FEAT-048). A private chat id is the owner's user id; + # ``record`` ignores anything that is not one, so a group summary is + # simply not filed anywhere. + try: + from condor.notifications import record, user_for_chat + + owner = user_for_chat(chat_id) + if owner: + await record(owner, text, kind="system") + except Exception: + logger.debug("Could not record interrupted-run notice", exc_info=True) async def startup(application: Application) -> None: @@ -882,10 +894,16 @@ async def _run_dual(application: Application) -> None: get_current_branch(), get_local_commit() ) version = f" ({branch} @ {commit})" if commit else "" + boot_text = f"Condor is online and ready.{version}" await application.bot.send_message( chat_id=int(ADMIN_USER_ID), - text=f"Condor is online and ready.{version}", + text=boot_text, ) + # The same notice on the dashboard bell (FEAT-048), so an admin who + # only has the browser open still sees which commit came up. + from condor.notifications import record + + await record(int(ADMIN_USER_ID), boot_text, kind="system") except Exception as e: logger.warning(f"Failed to send startup notification to admin: {e}") diff --git a/tests/conftest.py b/tests/conftest.py index 57edee04f..82768dfa0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -43,3 +43,17 @@ def _reset_gecko_throttle(): reset_gecko_throttle() yield + + +@pytest.fixture(autouse=True) +def _isolated_notification_store(tmp_path, monkeypatch): + """Keep the bell's store out of the developer's ``data/`` directory. + + ``condor.notifications.record`` is now reached from several producers + (FEAT-048), so without this any test that finishes a delegation, a routine + or a ``notify`` call appends to the real ``data/notifications.json`` — and a + test run would show up in the running install's notification bell. + """ + from condor import notifications + + monkeypatch.setattr(notifications, "_FILE", tmp_path / "notifications.json") diff --git a/tests/test_notifications.py b/tests/test_notifications.py new file mode 100644 index 000000000..474b8e02e --- /dev/null +++ b/tests/test_notifications.py @@ -0,0 +1,421 @@ +"""Task notifications reach the dashboard (FEAT-048). + +What is pinned here is the promise the feature makes: a finished background task +is *addressed to a user*, survives nobody watching, never crosses to another +user, and never costs its producer anything when the surface is down. + +Sync tests driving coroutines with ``asyncio.run``: ``pytest-asyncio`` is a dev +dependency but is not installed in this venv (same convention as +``test_notify_route.py``). +""" + +import asyncio +import json + +import pytest + +from condor import notifications +from condor.notifications import ( + NotifyBot, + list_for, + mark_read, + record, + unread_count, + user_for_chat, +) +from condor.web.models import WebUser + +USER_A = 111 +USER_B = 222 + + +@pytest.fixture +def store(tmp_path, monkeypatch): + """A private store file and an empty sink registry for each test.""" + path = tmp_path / "notifications.json" + monkeypatch.setattr(notifications, "_FILE", path) + monkeypatch.setattr(notifications, "_push_sinks", []) + return path + + +@pytest.fixture +def known_users(monkeypatch): + """Both test users exist as far as ``config_manager`` is concerned.""" + + class _CM: + def get_user(self, user_id): + return {"id": user_id} if user_id in (USER_A, USER_B) else None + + monkeypatch.setattr("config_manager.get_config_manager", lambda: _CM()) + + +# ── The store ── + + +def test_record_persists_and_reloads(store): + """A notification outlives the process that produced it.""" + n = asyncio.run(record(USER_A, "task done", kind="delegation")) + + assert n is not None + assert store.exists() + on_disk = json.loads(store.read_text()) + assert list(on_disk) == [str(USER_A)] + + items = list_for(USER_A) + assert [i.text for i in items] == ["task done"] + assert items[0].kind == "delegation" + assert items[0].read is False + assert unread_count(USER_A) == 1 + + +def test_newest_first_and_capped(store, monkeypatch): + """The bell keeps the recent past, not all of it.""" + monkeypatch.setattr(notifications, "_MAX_PER_USER", 5) + + async def fill(): + for i in range(8): + await record(USER_A, f"note {i}") + + asyncio.run(fill()) + + texts = [n.text for n in list_for(USER_A, limit=100)] + assert texts == ["note 7", "note 6", "note 5", "note 4", "note 3"] + + +def test_users_never_see_each_other(store): + """The store is per user, and reading is scoped to the reader.""" + + async def both(): + await record(USER_A, "A's task") + await record(USER_B, "B's task") + + asyncio.run(both()) + + assert [n.text for n in list_for(USER_A)] == ["A's task"] + assert [n.text for n in list_for(USER_B)] == ["B's task"] + + +def test_mark_read_cannot_reach_across_users(store): + """An id from someone else's bucket is simply not found.""" + b = asyncio.run(record(USER_B, "B's task")) + assert b is not None + + left = mark_read(USER_A, [b.id]) + + assert left == 0 # A has nothing of their own + assert unread_count(USER_B) == 1 + assert list_for(USER_B)[0].read is False + + +def test_mark_read_all_and_by_id(store): + async def three(): + return [await record(USER_A, f"note {i}") for i in range(3)] + + made = asyncio.run(three()) + + assert mark_read(USER_A, [made[0].id]) == 2 + assert mark_read(USER_A, None) == 0 + assert unread_count(USER_A) == 0 + + +def test_no_user_records_nothing(store): + """A producer with nobody behind it is not an error.""" + assert asyncio.run(record(0, "orphan")) is None + assert asyncio.run(record(None, "orphan")) is None + assert asyncio.run(record(USER_A, "")) is None + assert not store.exists() + + +def test_concurrent_records_do_not_lose_each_other(store): + """The write lock is what keeps a simultaneous finish from vanishing.""" + + async def both(): + await asyncio.gather( + *[record(USER_A, f"note {i}") for i in range(10)], + ) + + asyncio.run(both()) + assert len(list_for(USER_A, limit=100)) == 10 + + +# ── The push bus ── + + +def test_push_reaches_registered_sink(store): + seen: list = [] + + async def sink(n): + seen.append(n) + + notifications.register_push_sink(sink) + asyncio.run(record(USER_A, "live")) + + assert [n.text for n in seen] == ["live"] + + +def test_dead_sink_never_reaches_the_producer(store): + """A socket that blew up must not fail the task that was announcing itself.""" + + async def broken(n): + raise RuntimeError("socket is gone") + + notifications.register_push_sink(broken) + + n = asyncio.run(record(USER_A, "still recorded")) + + assert n is not None + assert [i.text for i in list_for(USER_A)] == ["still recorded"] + + +# ── The WS surface ── + + +class _FakeWS: + def __init__(self, explode=False): + self.sent: list[dict] = [] + self.explode = explode + + async def send_text(self, raw: str) -> None: + if self.explode: + raise RuntimeError("closed") + self.sent.append(json.loads(raw)) + + +def test_ws_push_reaches_only_the_addressed_user(store, monkeypatch): + from condor.web.routes import chat_ws + + monkeypatch.setattr(notifications, "_push_sinks", [chat_ws._push_notification]) + a_ws, b_ws = _FakeWS(), _FakeWS() + monkeypatch.setattr(chat_ws, "_attached_sockets", {USER_A: {a_ws}, USER_B: {b_ws}}) + + asyncio.run(record(USER_A, "A's task", kind="delegation", link="/routines")) + + assert b_ws.sent == [] + assert len(a_ws.sent) == 1 + frame = a_ws.sent[0] + assert frame["event"] == "notification" + assert frame["kind"] == "delegation" + assert frame["text"] == "A's task" + assert frame["link"] == "/routines" + assert frame["id"] and frame["ts"] + + +def test_ws_push_on_a_dead_socket_still_stores(store, monkeypatch): + from condor.web.routes import chat_ws + + monkeypatch.setattr(notifications, "_push_sinks", [chat_ws._push_notification]) + monkeypatch.setattr(chat_ws, "_attached_sockets", {USER_A: {_FakeWS(explode=True)}}) + + asyncio.run(record(USER_A, "unwatched")) + + assert [n.text for n in list_for(USER_A)] == ["unwatched"] + + +# ── The REST surface ── + + +def test_rest_is_scoped_to_the_jwt(store): + from condor.web.routes.notifications import ( + MarkReadRequest, + list_notifications, + mark_notifications_read, + ) + + async def both(): + await record(USER_A, "A's task") + b = await record(USER_B, "B's task") + return b + + b = asyncio.run(both()) + + listing = asyncio.run( + list_notifications(limit=50, user=WebUser(id=USER_A, role="user")) + ) + assert [i["text"] for i in listing["items"]] == ["A's task"] + assert listing["unread"] == 1 + + # A's session asking to read B's id changes nothing about B. + asyncio.run( + mark_notifications_read( + MarkReadRequest(ids=[b.id]), user=WebUser(id=USER_A, role="user") + ) + ) + assert unread_count(USER_B) == 1 + + +# ── The adapter and the outbound ladder ── + + +def test_notify_bot_records_a_send(store, known_users): + bot = NotifyBot() + + asyncio.run(bot.send_message(chat_id=USER_A, text="hello")) + + assert [n.text for n in list_for(USER_A)] == ["hello"] + + +def test_notify_bot_ignores_a_chat_with_no_owner(store, known_users): + bot = NotifyBot() + + asyncio.run(bot.send_message(chat_id=-100123, text="group chatter")) + asyncio.run(bot.send_message(chat_id=999999, text="stranger")) + + assert user_for_chat(-100123) is None + assert list_for(999999) == [] + assert not store.exists() + + +def test_notify_bot_documents_are_linked_not_rehosted(store, known_users): + bot = NotifyBot() + + asyncio.run(bot.send_document(chat_id=USER_A, document=b"x", filename="Daily.html")) + + item = list_for(USER_A)[0] + assert item.text == "Daily.html" + assert item.link == "/routines?tab=reports" + + +def test_resolve_bot_falls_through_to_the_bell_without_a_token( + store, known_users, monkeypatch +): + """With no Telegram at all, an outbound message becomes a bell item.""" + monkeypatch.delenv("TELEGRAM_TOKEN", raising=False) + monkeypatch.delenv("TELEGRAM_BOT_TOKEN", raising=False) + + class _NoBotStore: + def get_bot(self): + return None + + monkeypatch.setattr("condor.routine_store.get_routine_store", lambda: _NoBotStore()) + + from condor.agents.delegate import resolve_bot + + bot = resolve_bot() + assert isinstance(bot, NotifyBot) + + asyncio.run(bot.send_message(chat_id=USER_A, text="dropped no more")) + assert [n.text for n in list_for(USER_A)] == ["dropped no more"] + + +def test_resolve_bot_prefers_telegram_when_there_is_a_token(monkeypatch): + """A configured install is byte-for-byte unchanged.""" + monkeypatch.setenv("TELEGRAM_TOKEN", "123:abc") + + class _NoBotStore: + def get_bot(self): + return None + + monkeypatch.setattr("condor.routine_store.get_routine_store", lambda: _NoBotStore()) + + from condor.agents.delegate import resolve_bot + from condor.routine_store import _HttpBot + + assert isinstance(resolve_bot(), _HttpBot) + + +# ── Producers ── + + +def test_delegation_completion_lights_the_bell_once(store, known_users): + """One item, with exactly the text Telegram received.""" + from condor.agents.delegate import DelegateTask, _completion_text, _notify_done + + dt = DelegateTask( + task_id="t1", + agent_slug="condor", + user_id=USER_A, + chat_id=USER_A, + server_name=None, + task="do the thing", + status="done", + result="all done", + ) + + sent: list[dict] = [] + + class _Bot: + async def send_message(self, **kw): + sent.append(kw) + return {"ok": True} + + asyncio.run(_notify_done(dt, _Bot())) + + assert len(sent) == 1 + items = list_for(USER_A) + assert len(items) == 1 + assert items[0].text == _completion_text(dt) == sent[0]["text"] + assert items[0].kind == "delegation" + + +def test_delegation_over_the_bell_ladder_is_not_recorded_twice( + store, known_users, monkeypatch +): + """When the bell *is* the sender, ``_notify_done`` must not file it again.""" + monkeypatch.delenv("TELEGRAM_TOKEN", raising=False) + monkeypatch.delenv("TELEGRAM_BOT_TOKEN", raising=False) + + class _NoBotStore: + def get_bot(self): + return None + + monkeypatch.setattr("condor.routine_store.get_routine_store", lambda: _NoBotStore()) + + from condor.agents.delegate import DelegateTask, _notify_done + + dt = DelegateTask( + task_id="t2", + agent_slug="condor", + user_id=USER_A, + chat_id=USER_A, + server_name=None, + task="do the thing", + status="done", + result="all done", + ) + + asyncio.run(_notify_done(dt, None)) + + assert len(list_for(USER_A)) == 1 + + +def test_finished_routine_links_to_the_reports_tab(store): + """Even with no conversation behind the run, the owner is told.""" + from condor.routine_store import RoutineStore + + rs = RoutineStore() + rs._instances["i1"] = {"routine_name": "daily_pnl", "user_id": USER_A} + + asyncio.run(rs._report_run("i1", "12 pools checked", None)) + + item = list_for(USER_A)[0] + assert item.kind == "routine" + assert item.link == "/routines?tab=reports" + assert "daily_pnl" in item.text + + +def test_notify_route_counts_a_bell_only_delivery_as_recorded(store, monkeypatch): + """MCP ``send_notification`` succeeds on an install with no Telegram.""" + from condor.web.routes import agents as agents_routes + from condor.web.routes.agents import NotifyRequest, notify_user + + async def _no_conversation(session_key: str) -> str: + return "" + + monkeypatch.setattr(agents_routes, "_conversation_for_session", _no_conversation) + + class _DeadBot: + async def send_message(self, **kw): + return None + + monkeypatch.setattr("condor.agents.delegate.resolve_bot", lambda b=None: _DeadBot()) + + result = asyncio.run( + notify_user( + NotifyRequest(text="the thing happened", chat_id=USER_A), + user=WebUser(id=USER_A, role="user"), + ) + ) + + assert result["sent"] is False + assert result["recorded"] is True + assert [n.text for n in list_for(USER_A)] == ["the thing happened"] diff --git a/tests/test_notify_route.py b/tests/test_notify_route.py index 69998d4de..6e6566cb2 100644 --- a/tests/test_notify_route.py +++ b/tests/test_notify_route.py @@ -124,7 +124,10 @@ def test_a_dead_session_key_still_pushes_and_notes_nothing(monkeypatch, bot, not notify_user(NotifyRequest(text="ping", chat_id=1, session_key=""), user=CALLER) ) - assert result == {"sent": True, "recorded": False} + # No conversation to write into, so no transcript note -- but the dashboard + # bell is addressed to the user, not to a conversation, so it still records + # and the call still counts as delivered (FEAT-048). + assert result == {"sent": True, "recorded": True} assert notes == [] assert len(bot.calls) == 1 @@ -146,7 +149,9 @@ def explode(*a, **kw): ) ) - assert result == {"sent": True, "recorded": False} + # ``recorded`` is now true on the bell alone (FEAT-048): the transcript + # write failed, and that must not cost the user either delivery. + assert result == {"sent": True, "recorded": True} assert len(bot.calls) == 1 From be6a0cf55c66aa1a665092ade0dcef454a03a493 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Aug 2026 19:51:25 +0300 Subject: [PATCH 52/57] (feat) notification bell in the dashboard top bar A bell next to the Bug and Settings buttons, with an unread badge and a dropdown of the last 50 notices; an item carrying a link navigates there, and opening the list marks it read. Two sources, one react-query key: history from GET /notifications on mount, live arrivals written into the same cache by the chat socket's `notification` handler. So a notice that arrived while the tab was open and one that arrived an hour ago are indistinguishable, and a socket that was down costs nothing but latency. FEAT-048 --- frontend/src/components/layout/AppShell.tsx | 7 + .../components/layout/NotificationBell.tsx | 176 ++++++++++++++++++ frontend/src/hooks/useChatSocket.ts | 51 ++++- frontend/src/lib/api.ts | 32 ++++ 4 files changed, 265 insertions(+), 1 deletion(-) create mode 100644 frontend/src/components/layout/NotificationBell.tsx diff --git a/frontend/src/components/layout/AppShell.tsx b/frontend/src/components/layout/AppShell.tsx index a454f4e83..3ebc2bae9 100644 --- a/frontend/src/components/layout/AppShell.tsx +++ b/frontend/src/components/layout/AppShell.tsx @@ -24,6 +24,7 @@ import { usePrefetchData } from "@/hooks/usePrefetchData"; import { useServer } from "@/hooks/useServer"; import { useTheme } from "@/hooks/useTheme"; import { CurrencySelector } from "./CurrencySelector"; +import { NotificationBell } from "./NotificationBell"; import { ServerSelector } from "./ServerSelector"; const NAV_ITEMS = [ @@ -128,6 +129,12 @@ function AppShellBody() {
+ {/* Inside the ChatProvider on purpose: the bell reads the same + react-query cache the chat socket pushes live notifications + into, and that socket is what makes it update without a + reload (FEAT-048). */} + + + + {open && ( +
+
+ + Notifications + + {items.length > 0 && ( + + )} +
+ + {items.length === 0 && ( +
+ Nothing yet. Finished tasks show up here. +
+ )} + + {items.map((n) => { + const Icon = KIND_ICON[n.kind] ?? Bell; + return ( + + ); + })} +
+ )} +
+ ); +} diff --git a/frontend/src/hooks/useChatSocket.ts b/frontend/src/hooks/useChatSocket.ts index 4dcf83d45..59231fa59 100644 --- a/frontend/src/hooks/useChatSocket.ts +++ b/frontend/src/hooks/useChatSocket.ts @@ -1,5 +1,11 @@ import { useCallback, useEffect, useRef, useState } from "react"; -import { api, type ConversationTurn } from "@/lib/api"; +import { useQueryClient } from "@tanstack/react-query"; +import { + api, + type AppNotification, + type ConversationTurn, + type NotificationsResponse, +} from "@/lib/api"; import { useAuth } from "@/lib/auth"; import { toolCallState } from "@/lib/formatters"; import { getViewContext } from "@/lib/viewContext"; @@ -224,8 +230,19 @@ function turnsToMessages(turns: ConversationTurn[]): ChatMessage[] { */ const FLUSH_INTERVAL_MS = 50; +/** + * The bell's cache key (FEAT-048). + * + * Lives here because this is where the live `notification` event is written + * into it; `NotificationBell` reads the same key, so a pushed notice and a + * fetched one are one list. Not server-scoped: notifications belong to the + * user, not to whichever trading server is selected. + */ +export const NOTIFICATIONS_KEY = ["notifications"] as const; + export function useChatSocket() { const { token, user } = useAuth(); + const queryClient = useQueryClient(); const wsRef = useRef(null); const reconnectTimer = useRef>(undefined); // Whether this hook still wants a socket. `close()` is asynchronous, so the @@ -959,6 +976,37 @@ export function useChatSocket() { break; } + case "notification": { + // A background task finished (FEAT-048). Addressed to the *user*, not + // to a conversation, so it carries no slot and goes nowhere near the + // transcript — it lands in the bell's react-query cache, which is the + // same place `GET /notifications` fills on mount. Writing into the + // cache rather than into local state is what lets the bell be a leaf + // component with no wiring back up to here. + const incoming: AppNotification = { + id: (data.id as string) || "", + user_id: 0, + ts: (data.ts as number) || Date.now() / 1000, + kind: (data.kind as string) || "system", + text: (data.text as string) || "", + title: (data.title as string) || null, + link: (data.link as string) || null, + read: false, + }; + if (!incoming.id) break; + queryClient.setQueryData( + NOTIFICATIONS_KEY, + (prev) => { + // A reconnect can replay one we already hold; keyed by id so it + // is never listed twice. + const rest = (prev?.items ?? []).filter((n) => n.id !== incoming.id); + const items = [incoming, ...rest].slice(0, 50); + return { items, unread: items.filter((n) => !n.read).length }; + }, + ); + break; + } + case "heartbeat": break; } @@ -969,6 +1017,7 @@ export function useChatSocket() { flushChunks, hydrateSlot, prewarmLatest, + queryClient, send, stopStreaming, ], diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 90f3a3936..5c82a6a0a 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -14,6 +14,25 @@ async function apiFetch(path: string, init?: RequestInit): Promise { return res.json(); } +/** A finished background task, addressed to the user rather than to a chat. */ +export interface AppNotification { + id: string; + user_id: number; + ts: number; + /** "delegation" | "routine" | "agent" | "system" */ + kind: string; + text: string; + title?: string | null; + /** A dashboard route to open, e.g. "/routines?tab=reports". */ + link?: string | null; + read: boolean; +} + +export interface NotificationsResponse { + items: AppNotification[]; + unread: number; +} + // ── Types ── export interface ServerInfo { @@ -2213,4 +2232,17 @@ export const api = { `/api/v1/settings/custom-providers/${encodeURIComponent(name)}`, { method: "DELETE" }, ), + + // ── Notifications (FEAT-048) ── + + /** The bell's history. Scoped to the JWT server-side — there is no user param. */ + getNotifications: (limit = 50) => + apiFetch(`/api/v1/notifications?limit=${limit}`), + + /** Mark some notifications read, or all of them when `ids` is omitted. */ + markNotificationsRead: (ids?: string[]) => + apiFetch<{ unread: number }>("/api/v1/notifications/read", { + method: "POST", + body: JSON.stringify({ ids: ids ?? null }), + }), }; From 446e4b761c415359321db54f39a079b981281c35 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Aug 2026 20:08:57 +0300 Subject: [PATCH 53/57] (feat) condor runs without telegram: explicit CONDOR_MODE, local boot and /auth/local-login MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Telegram was both the runtime dependency and the identity provider: main() could not build an Application without a token, and the dashboard's only way to get a session was the one-time token /web mints. CONDOR_MODE=local now boots the process with no bot at all and logs the local admin straight in. The mode is explicit and is never inferred from whether a token happens to be present — telegram mode with an empty TELEGRAM_TOKEN exits at boot naming `make setup`, so an install that loses its token stops instead of quietly becoming a login-less dashboard. Local mode binds 127.0.0.1 only, with WEB_HOST as the one documented opt-out. Those two controls are independent and both required; tests pin each of them. Boot skips the PTB lifecycle rather than stubbing it: the Application is built on a placeholder token but never initialized, so nothing polls and no handler can dispatch, and the job queue is started directly (scheduled routines, update checks and signals all keep working). Outbound messages resolve to NotifyBot, so a routine's context.bot.send_message lands on the dashboard bell. Auth adds one endpoint. get_current_user, the WS handshake, check_server_access and the 200-odd guarded endpoints are untouched — /auth/local-login mints an ordinary JWT and 404s outside local mode, so nothing downstream ever learns which mode it is in. --- condor/web/routes/auth.py | 57 +++++++ main.py | 168 +++++++++++++++---- tests/test_local_mode.py | 338 ++++++++++++++++++++++++++++++++++++++ utils/config.py | 91 ++++++++++ 4 files changed, 618 insertions(+), 36 deletions(-) create mode 100644 tests/test_local_mode.py diff --git a/condor/web/routes/auth.py b/condor/web/routes/auth.py index 5eb6703d2..261ed2193 100644 --- a/condor/web/routes/auth.py +++ b/condor/web/routes/auth.py @@ -6,6 +6,7 @@ from condor.web.auth import create_jwt, get_current_user, redeem_login_token from condor.web.models import LoginResponse, WebUser from config_manager import UserRole, get_config_manager +from utils import config as app_config router = APIRouter(tags=["auth"]) @@ -55,6 +56,62 @@ async def token_login(req: TokenLoginRequest, response: Response): return LoginResponse(token=jwt_token, user=user) +@router.get("/auth/mode") +async def auth_mode() -> dict: + """How this install logs in. Unauthenticated on purpose. + + The Login page has to know *how* to log in before it can, and the answer is + a property of the install, not of any user: either "run /web in Telegram" or + "you are already home". It leaks nothing a visitor could not learn by trying + :func:`local_login` and reading the status code. + """ + return {"mode": app_config.CONDOR_MODE} + + +@router.post("/auth/local-login", response_model=LoginResponse) +async def local_login(response: Response): + """Mint a session for the local admin. Local mode only (FEAT-049). + + Deliberately the *only* mode-aware place in the auth path. Once this returns, + the JWT, the WS handshake, ``check_server_access``, roles and all 200-odd + guarded endpoints behave identically to Telegram mode, because they are + identical — ``get_current_user`` never learns which mode it is in. Outside + local mode this endpoint does not exist: 404, not 403, because "there is a + passwordless login here, you just may not use it" is not a thing worth + telling anyone. + + The control that keeps this safe is not this check alone — local mode also + binds loopback only (``utils.config.resolve_web_host``). + """ + if not app_config.LOCAL_MODE: + raise HTTPException(status_code=status.HTTP_404_NOT_FOUND, detail="Not found") + + # Same headers as token_login: never leak the issued JWT via Referer, and + # never let a proxy or the browser cache a session response. + response.headers["Referrer-Policy"] = "no-referrer" + response.headers["Cache-Control"] = "no-store" + + user_id = app_config.LOCAL_USER_ID + role = get_config_manager().get_user_role(user_id) + if role not in (UserRole.USER, UserRole.ADMIN): + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail=( + f"Local mode user {user_id} is not configured in config.yml — " + "re-run `make setup`." + ), + ) + + user = WebUser(id=user_id, username="local", first_name="Local", role=role.value) + token = create_jwt( + user_id=user_id, + username=user.username, + first_name=user.first_name, + role=role.value, + ) + return LoginResponse(token=token, user=user) + + @router.get("/auth/me", response_model=WebUser) async def me(user: WebUser = Depends(get_current_user)): return user diff --git a/main.py b/main.py index 7522a4f78..cc30454e7 100644 --- a/main.py +++ b/main.py @@ -23,7 +23,15 @@ from condor.telemetry import taps as telemetry_taps from handlers import cancel_command, clear_all_input_states from utils.auth import restricted -from utils.config import TELEGRAM_TOKEN, WEB_PORT, WEB_URL +from utils.config import ( + LOCAL_MODE, + TELEGRAM_TOKEN, + WEB_HOST, + WEB_PORT, + WEB_URL, + ConfigError, + check_startup_config, +) # Enable logging logging.basicConfig( @@ -516,6 +524,8 @@ async def _notify_interrupted_runs(bot, report) -> None: One summary per chat rather than a message per run: a crash with several live loops would otherwise spam the user at the worst possible moment. """ + from condor.notifications import NotifyBot + by_chat: dict[int, list] = {} for run in report.interrupted: status = None @@ -541,7 +551,10 @@ async def _notify_interrupted_runs(bot, report) -> None: logger.warning("Could not notify chat %s about interrupted runs", chat_id) # And on the bell (FEAT-048). A private chat id is the owner's user id; # ``record`` ignores anything that is not one, so a group summary is - # simply not filed anywhere. + # simply not filed anywhere. Skipped when the sender above *is* the bell + # (local mode, FEAT-049): it already filed exactly this text. + if isinstance(bot, NotifyBot): + continue try: from condor.notifications import record, user_for_chat @@ -552,6 +565,23 @@ async def _notify_interrupted_runs(bot, report) -> None: logger.debug("Could not record interrupted-run notice", exc_info=True) +def _outbound_bot(application: Application): + """The object this process sends user-facing messages through. + + Telegram mode: the real PTB bot, exactly as before. Local mode: there is no + Telegram behind the placeholder token, so outbound messages go to the + dashboard bell instead (:class:`condor.notifications.NotifyBot`, FEAT-048). + Everything downstream — the routine store, the session health monitor, the + interrupted-run summaries — keeps calling ``send_message`` and neither knows + nor cares which surface it reached. + """ + if not LOCAL_MODE: + return application.bot + from condor.notifications import NotifyBot + + return NotifyBot() + + async def startup(application: Application) -> None: """Bring the process up: commands, caches, supervisors, boot reconciliation. @@ -572,8 +602,16 @@ async def startup(application: Application) -> None: asyncio.get_event_loop().run_in_executor(None, _get_model, DEFAULT_MODEL) - # Register command menus (public + admin overlay) - await register_bot_commands(application) + # Whatever this process pushes at users from here on. In local mode there is + # no Telegram to push to, so it is the dashboard bell (FEAT-048) instead — + # which is what keeps the documented "context.bot is never None" contract + # true for routines with no bot behind them. + outbound_bot = _outbound_bot(application) + + # Register command menus (public + admin overlay). Pure Telegram: there is + # no command menu to publish when nothing polls. + if not LOCAL_MODE: + await register_bot_commands(application) # Restore scheduled routine jobs from persistence from handlers.routines import restore_scheduled_jobs @@ -583,7 +621,7 @@ async def startup(application: Application) -> None: # Inject Telegram bot into routine store so web-triggered routines can send messages from condor.routine_store import get_routine_store - get_routine_store().set_bot(application.bot) + get_routine_store().set_bot(outbound_bot) # Start ServerDataService (unified server-centric cache) from condor.server_data_service import get_server_data_service @@ -600,7 +638,7 @@ async def startup(application: Application) -> None: from condor.runtime import sessions as runtime_sessions from condor.runtime.confirmations import get_registry - await runtime_sessions.start_health_monitor(application.bot) + await runtime_sessions.start_health_monitor(outbound_bot) # Sweeps expired approvals so a request nobody answers is denied, not leaked. await get_registry().start() @@ -616,7 +654,7 @@ async def startup(application: Application) -> None: report.total, len(report.restarted), ) - await _notify_interrupted_runs(application.bot, report) + await _notify_interrupted_runs(outbound_bot, report) except Exception: logger.exception("Boot reconciliation failed; continuing startup") @@ -802,6 +840,15 @@ async def send_to_all(self, message: str, parse_mode: str = "Markdown"): def main() -> None: """Run the bot.""" + # Refuse to start on a configuration that cannot mean what it says: telegram + # mode (the default) with no token used to surface as an InvalidToken + # traceback from inside PTB, and must never be quietly read as "local mode". + try: + check_startup_config() + except ConfigError as exc: + logger.error("%s", exc) + raise SystemExit(1) + # Reap any ACP/MCP subprocess trees orphaned by a prior hard kill (kill -9, # OOM, power loss) before we spawn our own — those bypass teardown(). try: @@ -820,9 +867,12 @@ def main() -> None: # Create the Application with persistence enabled. No post_init/post_shutdown # hooks: they only fire from run_polling/run_webhook, and _run_dual owns the # lifecycle — startup() and teardown() are called there, explicitly. + # In local mode there is no token and nothing polls; the placeholder exists + # only so the Application (and with it job_queue, CallbackContext and the + # handler registry) can be built at all. Nothing ever calls Telegram with it. application = ( Application.builder() - .token(TELEGRAM_TOKEN) + .token(TELEGRAM_TOKEN or "0:local") .persistence(persistence) .concurrent_updates(True) .build() @@ -849,6 +899,25 @@ def main() -> None: exec_restart() # never returns +def _web_server_config(web_app): + """uvicorn's config for the dashboard, isolated so the bind address is testable. + + ``WEB_HOST`` is ``0.0.0.0`` in telegram mode (unchanged) and loopback in + local mode, where the dashboard has no login at all — see + :func:`utils.config.resolve_web_host` for why that is not negotiable by + accident. + """ + import uvicorn + + return uvicorn.Config( + web_app, + host=WEB_HOST, + port=WEB_PORT, + log_level="info", + access_log=False, + ) + + async def _run_dual(application: Application) -> None: """Run the Telegram bot and FastAPI web server concurrently.""" import signal @@ -862,21 +931,24 @@ async def _run_dual(application: Application) -> None: # initialize() and start_polling() — the same slot PTB gives post_init — so # commands, caches and boot reconciliation are settled before the first # update is dispatched. - await application.initialize() - await startup(application) - await application.updater.start_polling(allowed_updates=Update.ALL_TYPES) - await application.start() + # + # Local mode skips that lifecycle rather than faking it (FEAT-049): the + # Application is built but never initialized, so nothing polls and no handler + # can dispatch — the Telegram surface is inert, not mocked. The job queue is + # started directly, which is all scheduled routines, update checks and + # signals actually need. + if LOCAL_MODE: + await startup(application) + await application.job_queue.start() + else: + await application.initialize() + await startup(application) + await application.updater.start_polling(allowed_updates=Update.ALL_TYPES) + await application.start() # Create and start the web server web_app = create_app() - config = uvicorn.Config( - web_app, - host="0.0.0.0", - port=WEB_PORT, - log_level="info", - access_log=False, - ) - server = uvicorn.Server(config) + server = uvicorn.Server(_web_server_config(web_app)) # Start WebSocket manager get_ws_manager().start() @@ -895,10 +967,11 @@ async def _run_dual(application: Application) -> None: ) version = f" ({branch} @ {commit})" if commit else "" boot_text = f"Condor is online and ready.{version}" - await application.bot.send_message( - chat_id=int(ADMIN_USER_ID), - text=boot_text, - ) + if not LOCAL_MODE: + await application.bot.send_message( + chat_id=int(ADMIN_USER_ID), + text=boot_text, + ) # The same notice on the dashboard bell (FEAT-048), so an admin who # only has the browser open still sees which commit came up. from condor.notifications import record @@ -909,12 +982,25 @@ async def _run_dual(application: Application) -> None: # Ask, once, whether this install wants to be counted (FEAT-023). Sent # next to the boot notification because that is the one moment the admin - # is already looking. Until it is answered, nothing is collected. - from condor.telemetry.prompt import maybe_prompt_admin - - await maybe_prompt_admin(application.bot) - - logger.info("Starting Condor: Telegram bot + web dashboard on port %s", WEB_PORT) + # is already looking. Until it is answered, nothing is collected. The + # prompt is a Telegram message with inline buttons, so local mode has + # nowhere to ask and stays silent — which leaves consent `unknown`, + # which emits nothing. + if not LOCAL_MODE: + from condor.telemetry.prompt import maybe_prompt_admin + + await maybe_prompt_admin(application.bot) + + if LOCAL_MODE: + logger.info( + "Starting Condor in local mode (no Telegram): dashboard on http://%s:%s", + WEB_HOST, + WEB_PORT, + ) + else: + logger.info( + "Starting Condor: Telegram bot + web dashboard on port %s", WEB_PORT + ) # Handle shutdown signals shutdown_event = asyncio.Event() @@ -945,12 +1031,22 @@ def _signal_handler(): logger.exception("Web server crashed during shutdown") # Run each step independently so one failure can't skip the rest. # teardown() runs last, mirroring where post_shutdown would have sat. - for name, step in ( - ("updater.stop", application.updater.stop), - ("application.stop", application.stop), - ("application.shutdown", application.shutdown), - ("teardown", partial(teardown, application)), - ): + # Local mode never initialized or started the Application, so the PTB + # stop steps would only raise "not running" — it has a job queue to stop + # and nothing else. + if LOCAL_MODE: + steps = ( + ("job_queue.stop", application.job_queue.stop), + ("teardown", partial(teardown, application)), + ) + else: + steps = ( + ("updater.stop", application.updater.stop), + ("application.stop", application.stop), + ("application.shutdown", application.shutdown), + ("teardown", partial(teardown, application)), + ) + for name, step in steps: try: await step() except Exception: diff --git a/tests/test_local_mode.py b/tests/test_local_mode.py new file mode 100644 index 000000000..8fa2cf409 --- /dev/null +++ b/tests/test_local_mode.py @@ -0,0 +1,338 @@ +"""Condor runs without Telegram (FEAT-049). + +Two of these tests are the feature's safety controls and exist to fail loudly: + +* **Mode is never inferred.** ``CONDOR_MODE`` unset with an empty + ``TELEGRAM_TOKEN`` is a hard error naming ``make setup`` — an install that + loses its token must stop, never quietly become an unauthenticated dashboard. +* **Local mode binds loopback only.** ``WEB_HOST`` is the one, explicit, + documented way to expose a dashboard that has no login at all. + +Everything else pins the seams: the local-login endpoint exists only in local +mode, the JWT it mints is an ordinary one (``get_current_user`` never learns +which mode it is in), the identity ``make setup`` writes actually resolves, and +PTB still runs a job queue on an Application that was never initialized. + +Sync tests driving coroutines with ``asyncio.run``: ``pytest-asyncio`` is a dev +dependency but is not installed in this venv (same convention as +``tests/test_notifications.py``). +""" + +import asyncio +import ipaddress + +import pytest +from starlette.testclient import TestClient + +from condor.web.app import create_app +from config_manager import ConfigManager, ServerPermission, UserRole +from utils import config as app_config +from utils.config import ( + ConfigError, + check_startup_config, + resolve_local_user_id, + resolve_mode, + resolve_web_host, +) + +LOCAL_ID = 1 + + +# ── Mode resolution: explicit, never inferred ── + + +@pytest.mark.parametrize( + "env, expected", + [ + ({}, "telegram"), + ({"CONDOR_MODE": ""}, "telegram"), + ({"CONDOR_MODE": " "}, "telegram"), + ({"CONDOR_MODE": "telegram"}, "telegram"), + ({"CONDOR_MODE": "local"}, "local"), + ({"CONDOR_MODE": "LOCAL"}, "local"), + ({"CONDOR_MODE": " Local "}, "local"), + # A typo must not land in local mode. Telegram is the fail-safe answer: + # it then demands a token, which is the hard error below. + ({"CONDOR_MODE": "locl"}, "telegram"), + # The presence of a token says nothing about the mode, in either + # direction: this is the inference that must never happen. + ({"TELEGRAM_TOKEN": ""}, "telegram"), + ({"CONDOR_MODE": "local", "TELEGRAM_TOKEN": "123:abc"}, "local"), + ], +) +def test_resolve_mode(env, expected): + assert resolve_mode(env) == expected + + +def test_no_mode_and_no_token_is_a_hard_error_naming_setup(): + """CONTROL (a). The failure this feature must never have. + + An install whose token went missing has to stop with an instruction, not + fall through to a dashboard anybody can walk into. + """ + with pytest.raises(ConfigError) as excinfo: + check_startup_config({}) + + message = str(excinfo.value) + assert "make setup" in message + assert "CONDOR_MODE" in message + + +@pytest.mark.parametrize( + "env", + [ + {}, + {"TELEGRAM_TOKEN": ""}, + {"TELEGRAM_TOKEN": " "}, + {"CONDOR_MODE": "telegram"}, + {"CONDOR_MODE": "telegram", "TELEGRAM_TOKEN": ""}, + # A mistyped mode is telegram, so it still needs a token. + {"CONDOR_MODE": "locall", "TELEGRAM_TOKEN": ""}, + ], +) +def test_telegram_mode_without_a_token_refuses_to_start(env): + """CONTROL (a), across the whole env matrix that could mean 'no token'.""" + with pytest.raises(ConfigError): + check_startup_config(env) + + +@pytest.mark.parametrize( + "env", + [ + {"TELEGRAM_TOKEN": "123:abc"}, + {"CONDOR_MODE": "telegram", "TELEGRAM_TOKEN": "123:abc"}, + # Local mode is the only way to legitimately run with no token. + {"CONDOR_MODE": "local"}, + {"CONDOR_MODE": "local", "TELEGRAM_TOKEN": ""}, + ], +) +def test_valid_configurations_start(env): + check_startup_config(env) # must not raise + + +# ── Bind address ── + + +def test_local_mode_binds_loopback_only(): + """CONTROL (b). Local mode has no authentication; it must not be routable.""" + host = resolve_web_host({"CONDOR_MODE": "local"}) + + assert ipaddress.ip_address(host).is_loopback + assert host == "127.0.0.1" + + +def test_telegram_mode_still_binds_all_interfaces(): + """Telegram mode authenticates, and its binding is unchanged.""" + assert resolve_web_host({}) == "0.0.0.0" + assert resolve_web_host({"CONDOR_MODE": "telegram"}) == "0.0.0.0" + + +@pytest.mark.parametrize("mode", ["local", "telegram", ""]) +def test_web_host_is_the_explicit_opt_out(mode): + """CONTROL (b)'s escape hatch: exposing it has to be a deliberate act.""" + env = {"CONDOR_MODE": mode, "WEB_HOST": "0.0.0.0"} + assert resolve_web_host(env) == "0.0.0.0" + + env["WEB_HOST"] = "10.0.0.5" + assert resolve_web_host(env) == "10.0.0.5" + + +def test_uvicorn_binds_the_resolved_host(monkeypatch): + """CONTROL (b) is wired, not just computed: uvicorn gets WEB_HOST.""" + import main + + monkeypatch.setattr(main, "WEB_HOST", "127.0.0.1") + assert ipaddress.ip_address(main._web_server_config(object()).host).is_loopback + + monkeypatch.setattr(main, "WEB_HOST", "0.0.0.0") + assert main._web_server_config(object()).host == "0.0.0.0" + + +# ── Local identity ── + + +@pytest.mark.parametrize( + "env, expected", + [ + ({}, 1), + ({"CONDOR_LOCAL_USER_ID": ""}, 1), + ({"CONDOR_LOCAL_USER_ID": "42"}, 42), + # Junk and falsy ids fall back to 1: several call sites read a falsy + # user or chat id as "absent", so 0 would silently address nobody. + ({"CONDOR_LOCAL_USER_ID": "nope"}, 1), + ({"CONDOR_LOCAL_USER_ID": "0"}, 1), + ({"CONDOR_LOCAL_USER_ID": "-5"}, 1), + ], +) +def test_resolve_local_user_id(env, expected): + assert resolve_local_user_id(env) == expected + + +def test_setup_template_gives_the_local_user_a_working_identity(tmp_path, monkeypatch): + """What ``make setup`` writes in local mode has to actually resolve. + + ``ADMIN_USER_ID=1`` in ``.env`` plus ``admin_id: 1`` and + ``server_access.local.owner_id: 1`` in ``config.yml`` is the whole identity + story — no new concept, just the integer everything already keys on. If this + breaks, ``/auth/local-login`` 500s and nothing else works. + + The ``.env`` half is load-bearing, not decoration: ``ConfigManager`` only + materialises the admin's *user record* from the environment's + ``ADMIN_USER_ID`` (``_get_admin_from_env``), so a ``config.yml`` naming + ``admin_id: 1`` with an empty ``users:`` grants nothing on its own. + """ + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(app_config, "ADMIN_USER_ID", LOCAL_ID) + (tmp_path / "config.yml").write_text( + """servers: + local: + host: localhost + port: 8000 + username: admin + password: admin + +default_server: local + +admin_id: 1 + +users: {} + +server_access: + local: + owner_id: 1 + created_at: null + shared_with: {} + +chat_defaults: + 1: local + +version: 1 +""", + encoding="utf-8", + ) + + cm = ConfigManager(config_path=str(tmp_path / "config.yml")) + + assert cm.get_user_role(LOCAL_ID) == UserRole.ADMIN + assert cm.has_server_access(LOCAL_ID, "local", ServerPermission.TRADER) + + +# ── The one mode-aware endpoint ── + + +@pytest.fixture +def client(monkeypatch): + """A dashboard whose config manager knows exactly one user: the local admin.""" + monkeypatch.setenv("WEB_JWT_SECRET", "test-secret-for-local-mode") + + class _CM: + def get_user_role(self, user_id): + return UserRole.ADMIN if user_id == LOCAL_ID else None + + monkeypatch.setattr("condor.web.routes.auth.get_config_manager", lambda: _CM()) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: _CM()) + monkeypatch.setattr(app_config, "LOCAL_USER_ID", LOCAL_ID) + with TestClient(create_app()) as test_client: + yield test_client + + +def _set_mode(monkeypatch, mode: str) -> None: + monkeypatch.setattr(app_config, "CONDOR_MODE", mode) + monkeypatch.setattr(app_config, "LOCAL_MODE", mode == "local") + + +def test_local_login_does_not_exist_in_telegram_mode(client, monkeypatch): + """CONTROL, second half: no passwordless login exists where anyone can reach it.""" + _set_mode(monkeypatch, "telegram") + assert client.post("/api/v1/auth/local-login").status_code == 404 + + +def test_local_login_does_not_exist_when_mode_is_unset(client, monkeypatch): + """An install that never heard of CONDOR_MODE has no local login.""" + _set_mode(monkeypatch, resolve_mode({})) + assert client.post("/api/v1/auth/local-login").status_code == 404 + + +def test_local_login_mints_a_session_the_shared_guard_accepts(client, monkeypatch): + """The JWT is an ordinary one: ``get_current_user`` never learns the mode.""" + _set_mode(monkeypatch, "local") + + res = client.post("/api/v1/auth/local-login") + assert res.status_code == 200 + body = res.json() + assert body["user"]["id"] == LOCAL_ID + assert body["user"]["role"] == "admin" + assert res.headers["Referrer-Policy"] == "no-referrer" + assert res.headers["Cache-Control"] == "no-store" + + # The unchanged guard, reached through an unchanged endpoint. + me = client.get( + "/api/v1/auth/me", headers={"Authorization": f"Bearer {body['token']}"} + ) + assert me.status_code == 200 + assert me.json()["id"] == LOCAL_ID + + +def test_local_login_refuses_an_unconfigured_local_user(client, monkeypatch): + """No config.yml entry means no session — not a session for a stranger.""" + _set_mode(monkeypatch, "local") + monkeypatch.setattr(app_config, "LOCAL_USER_ID", 4242) + + res = client.post("/api/v1/auth/local-login") + assert res.status_code == 500 + assert "make setup" in res.json()["detail"] + + +def test_auth_mode_reports_the_mode_without_a_session(client, monkeypatch): + """The Login page has to know how to log in before it can.""" + _set_mode(monkeypatch, "local") + assert client.get("/api/v1/auth/mode").json() == {"mode": "local"} + + _set_mode(monkeypatch, "telegram") + assert client.get("/api/v1/auth/mode").json() == {"mode": "telegram"} + + +# ── Boot without Telegram ── + + +def test_job_queue_runs_on_an_uninitialized_application(): + """Local mode's whole boot rests on this PTB behaviour, so pin it. + + Scheduled routines, update checks and signals are all ``job_queue`` jobs. + Local mode never calls ``initialize()``/``start()`` (nothing polls), it just + starts the queue — a PTB release that breaks that must fail here rather than + in someone's install. + """ + from telegram.ext import Application + + async def scenario(): + application = Application.builder().token("0:local").build() + fired = asyncio.Event() + + async def _job(_context): + fired.set() + + # Scheduled *before* the queue starts, exactly as ``startup()`` does. + application.job_queue.run_once(_job, 0) + await application.job_queue.start() + try: + await asyncio.wait_for(fired.wait(), timeout=5) + finally: + await application.job_queue.stop() + + asyncio.run(scenario()) + + +def test_outbound_messages_go_to_the_bell_in_local_mode(monkeypatch): + """``context.bot`` is never None: with no Telegram it is the dashboard bell.""" + import main + from condor.notifications import NotifyBot + + class _App: + bot = object() + + monkeypatch.setattr(main, "LOCAL_MODE", True) + assert isinstance(main._outbound_bot(_App()), NotifyBot) + + monkeypatch.setattr(main, "LOCAL_MODE", False) + assert main._outbound_bot(_App()) is _App.bot diff --git a/utils/config.py b/utils/config.py index 0430115ec..c43fd57da 100644 --- a/utils/config.py +++ b/utils/config.py @@ -41,3 +41,94 @@ # ping - install / heartbeat / version_change / shutdown only. # usage - the full allowlisted taxonomy in condor/telemetry/schema.py. CONDOR_TELEMETRY = os.environ.get("CONDOR_TELEMETRY", "").strip().lower() or None + +# ── Run mode (FEAT-049) ── +# Condor runs either as a Telegram bot (the default) or as a local, Telegram-less +# dashboard. The mode is *explicit*: it is read from CONDOR_MODE and is never +# inferred from whether a token happens to be present. An install whose +# TELEGRAM_TOKEN went missing must fail loudly at boot, not quietly become an +# unauthenticated dashboard — local mode has no authentication at all. + + +class ConfigError(RuntimeError): + """The process is configured in a way it must not start with.""" + + +MODE_TELEGRAM = "telegram" +MODE_LOCAL = "local" + + +def resolve_mode(env=None) -> str: + """The configured run mode: ``"telegram"`` (default) or ``"local"``. + + Only the exact value ``local`` (case- and space-insensitive) selects local + mode. Anything else — unset, empty, or a typo — is ``telegram``, which is + the fail-safe direction: telegram mode requires a token, so a mistyped mode + surfaces as the hard error below instead of as an open dashboard. + """ + env = os.environ if env is None else env + value = (env.get("CONDOR_MODE") or "").strip().lower() + return MODE_LOCAL if value == MODE_LOCAL else MODE_TELEGRAM + + +def resolve_web_host(env=None) -> str: + """The address the dashboard binds to. + + Local mode has no login, so it binds loopback only: an unauthenticated + dashboard with full trading control must not be one firewall rule away from + the internet. ``WEB_HOST`` is the explicit, documented opt-out (set it to + ``0.0.0.0`` and you have chosen to expose it). Telegram mode, which does + authenticate, keeps binding all interfaces. + """ + env = os.environ if env is None else env + explicit = (env.get("WEB_HOST") or "").strip() + if explicit: + return explicit + return "127.0.0.1" if resolve_mode(env) == MODE_LOCAL else "0.0.0.0" + + +def resolve_local_user_id(env=None) -> int: + """The user id local mode logs in as. ``1`` unless told otherwise. + + Local mode deliberately introduces no new identity concept: ``config.yml``'s + ``admin_id``, ``server_access.owner_id``, ``chat_defaults``, preferences, + memory and session keys all key on an integer user id today and keep working + untouched. ``1`` and not ``0`` because several call sites test a user or chat + id for truthiness, where a falsy id reads as "absent". + """ + env = os.environ if env is None else env + raw = (env.get("CONDOR_LOCAL_USER_ID") or "").strip() + if raw: + try: + value = int(raw) + if value > 0: + return value + except ValueError: + pass + return 1 + + +def check_startup_config(env=None) -> None: + """Refuse to start on a configuration that cannot mean what it says. + + The one case that matters: telegram mode (including the default, i.e. an + unset ``CONDOR_MODE``) with no token. Today that surfaces deep inside PTB as + an ``InvalidToken`` traceback; more importantly it must *never* be treated + as "well, local mode then". + """ + env = os.environ if env is None else env + if ( + resolve_mode(env) != MODE_LOCAL + and not (env.get("TELEGRAM_TOKEN") or "").strip() + ): + raise ConfigError( + "TELEGRAM_TOKEN is not set and CONDOR_MODE is not 'local'.\n" + "Run `make setup` to configure a Telegram bot, or choose Local mode " + "there (CONDOR_MODE=local) to run the dashboard without Telegram." + ) + + +CONDOR_MODE = resolve_mode() +LOCAL_MODE = CONDOR_MODE == MODE_LOCAL +WEB_HOST = resolve_web_host() +LOCAL_USER_ID = resolve_local_user_id() From ac0544b69756d9364fb2bced3efc096b6ab67874 Mon Sep 17 00:00:00 2001 From: cardosofede Date: Thu, 20 Aug 2026 20:09:07 +0300 Subject: [PATCH 54/57] (feat) local mode setup question, dashboard auto-login and docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Setup opens with the choice — Telegram (today's prompts, verbatim, plus an explicit CONDOR_MODE=telegram) or Local, which asks nothing: it writes CONDOR_MODE=local, ADMIN_USER_ID=1 and the localhost WEB_URL, and prints what local mode costs. An install from before this existed has a token but no CONDOR_MODE; the next run records telegram for it, so the mode is always explicit on disk. The Login page asks the server how this install authenticates before it renders, so the "run /web in Telegram" card never flashes on a machine that has no Telegram, and local mode lands straight on the dashboard. README documents local mode as what it is: no login, loopback only, and WEB_HOST=0.0.0.0 as a deliberate act with a warning attached. --- README.md | 35 +++++++++++++-- frontend/src/lib/auth.ts | 21 +++++++++ frontend/src/pages/Login.tsx | 40 +++++++++++++++++- setup-environment.sh | 82 ++++++++++++++++++++++++++++++++++-- 4 files changed, 170 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 57ac83698..ab7f4237e 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ Open Terminal, go to an **empty folder** where you are happy to create files (fo curl -fsSL https://raw.githubusercontent.com/hummingbot/deploy/main/setup.sh | bash ``` -The installer walks you through setup—for example your **Telegram** bot token and your **Telegram user id**—and can also install **Hummingbot API** on the **same machine** if you choose that when it asks. +The installer first asks **how you will use Condor**: **Telegram** (a bot you control from your phone — recommended) or **Local** (no Telegram at all, see [Local mode](#local-mode-no-telegram) below). If you choose Telegram it asks for your bot token and your Telegram user id. Either way it can also install **Hummingbot API** on the **same machine** if you choose that when it asks. When installing Hummingbot API, answer **`y`** when asked to enable Tailscale and paste your auth key. When it finishes, continue to **After installation** below. @@ -97,6 +97,29 @@ The following applies after **Install Condor**. If you used **Install only Hummi - If Condor and the API are on **different machines**, install [Tailscale](https://tailscale.com/download) on the Condor host and add the API in **`/servers`** with host **`hummingbot-api`** (not a public IP). See [Secure Connection via Tailscale](#secure-connection-via-tailscale) below. - If something fails, see **Troubleshooting** below. +## Local mode (no Telegram) + +Answer **2) Local** at the first setup question and Condor runs with **no Telegram bot at all**: no token, no `/web` login link, no polling. `make run` boots the process and the dashboard is at **http://localhost:8088**, already logged in. Everything else is the same product — portfolio, bots, agents, routines, trading — and notifications that would have gone to Telegram land in the dashboard's notification bell instead. + +Use it for development, or to try Condor before deciding whether you want a bot. + +> **Local mode has no login.** Anyone who can reach the port has full trading +> control. That is why it binds **`127.0.0.1` only** — the dashboard is reachable +> from the machine it runs on and nowhere else, not from your LAN and not from +> the internet. +> +> `WEB_HOST=0.0.0.0` in `.env` is the one, deliberate way to expose it. Only set +> it behind something that authenticates for you (Tailscale, an SSH tunnel, an +> authenticating reverse proxy). If you want Condor reachable from your phone, +> the answer is usually Telegram mode, not this. + +The mode is **explicit** and is never guessed from whether a token is present: an +install configured for Telegram whose `TELEGRAM_TOKEN` goes missing **exits at +boot** telling you to run `make setup`. It never falls back to a login-less +dashboard. + +Switching later is a `make setup` re-run (or editing `CONDOR_MODE` in `.env`). + ## Commands | Command | Description | @@ -234,10 +257,14 @@ Preferences are automatically saved and persist across sessions: ### `.env` ```bash -TELEGRAM_TOKEN=your_bot_token -ADMIN_USER_ID=123456789 +CONDOR_MODE=telegram # telegram (default) or local — see Local mode +TELEGRAM_TOKEN=your_bot_token # Required unless CONDOR_MODE=local +ADMIN_USER_ID=123456789 # Your Telegram user id (local mode uses 1) OPENAI_API_KEY=sk-... # Optional, for AI features OPENROUTER_API_KEY=sk-or-... # Optional, unlocks the OpenRouter LLM picker +WEB_HOST=0.0.0.0 # Optional. Overrides the dashboard bind + # address. Local mode binds 127.0.0.1; + # read the Local mode warning first. ``` > **OpenRouter:** Add `OPENROUTER_API_KEY` to `.env`, then in `/agent → Change LLM` @@ -371,6 +398,8 @@ Both `condor` and `hummingbot-api` should appear as connected peers. | Issue | Solution | |-------|----------| | Bot not responding | Check `TELEGRAM_TOKEN` and `ADMIN_USER_ID` in `.env` | +| Exits at boot: `TELEGRAM_TOKEN is not set` | Telegram mode needs a token. Run `make setup` — or choose Local mode there if you do not want a bot | +| Local mode: dashboard not reachable from another device | Intended — local mode binds `127.0.0.1`. See [Local mode](#local-mode-no-telegram) before changing `WEB_HOST` | | Access pending | Admin must approve user via /config > Admin Panel | | Commands failing | Verify Hummingbot API is running | | Connection refused | Check server host:port in `/servers`; use `hummingbot-api` (not `localhost`) when API is on another machine via Tailscale | diff --git a/frontend/src/lib/auth.ts b/frontend/src/lib/auth.ts index 81dc17670..476024770 100644 --- a/frontend/src/lib/auth.ts +++ b/frontend/src/lib/auth.ts @@ -21,6 +21,8 @@ export interface AuthState { token: string | null; isAuthenticated: boolean; loginWithToken: (loginToken: string) => Promise; + /** Local mode (FEAT-049): claim the local admin's session. 404s elsewhere. */ + loginLocal: () => Promise; logout: () => void; } @@ -34,6 +36,7 @@ export const AuthContext = createContext({ token: null, isAuthenticated: false, loginWithToken: async () => {}, + loginLocal: async () => {}, logout: () => {}, }); @@ -67,6 +70,23 @@ export function useAuthState(): AuthState { setUser(data.user); }, []); + // Local mode: the dashboard runs on this machine with no login at all, so + // the session is claimed rather than presented. Stores exactly what + // loginWithToken stores — downstream there is no such thing as a "local" + // session, only a session. + const loginLocal = useCallback(async () => { + const res = await fetch("/api/v1/auth/local-login", { method: "POST" }); + if (!res.ok) { + const err = await res.json().catch(() => ({})); + throw new Error(err.detail || "Local login failed"); + } + const data = await res.json(); + localStorage.setItem(TOKEN_KEY, data.token); + localStorage.setItem(USER_KEY, JSON.stringify(data.user)); + setToken(data.token); + setUser(data.user); + }, []); + const logout = useCallback(() => { localStorage.removeItem(TOKEN_KEY); localStorage.removeItem(USER_KEY); @@ -99,6 +119,7 @@ export function useAuthState(): AuthState { token, isAuthenticated: !!token && !!user, loginWithToken, + loginLocal, logout, }; } diff --git a/frontend/src/pages/Login.tsx b/frontend/src/pages/Login.tsx index b8d1bab75..214672ca0 100644 --- a/frontend/src/pages/Login.tsx +++ b/frontend/src/pages/Login.tsx @@ -4,12 +4,16 @@ import { useNavigate, useSearchParams } from "react-router-dom"; import { useAuth } from "@/lib/auth"; export function Login() { - const { isAuthenticated, loginWithToken } = useAuth(); + const { isAuthenticated, loginWithToken, loginLocal } = useAuth(); const navigate = useNavigate(); const [searchParams] = useSearchParams(); const [error, setError] = useState(""); const [loggingIn, setLoggingIn] = useState(false); const attempted = useRef(false); + // How this install logs in — asked once, before anything is rendered, so the + // "run /web in Telegram" card never flashes on a machine that has no Telegram. + const [mode, setMode] = useState(null); + const probed = useRef(false); // Where to land after login. Only allow internal paths to avoid open redirects. const rawRedirect = searchParams.get("redirect"); @@ -44,12 +48,44 @@ export function Login() { }); }, [searchParams, loginWithToken, navigate, redirectTo]); + // Local mode (FEAT-049): there is no login step. Ask the server how it + // authenticates and, if the answer is "locally", claim the session and go. + // Skipped entirely when a one-time ?token= is being redeemed above. + useEffect(() => { + if (searchParams.get("token") || probed.current) return; + probed.current = true; + + let cancelled = false; + fetch("/api/v1/auth/mode") + .then((res) => (res.ok ? res.json() : { mode: "telegram" })) + .then((data) => { + if (cancelled) return; + setMode(data?.mode ?? "telegram"); + if (data?.mode !== "local") return; + setLoggingIn(true); + return loginLocal() + .then(() => navigate(redirectTo, { replace: true })) + .catch((err) => { + setError(err instanceof Error ? err.message : "Login failed"); + setLoggingIn(false); + }); + }) + .catch(() => { + // The server is unreachable, not Telegram-less: show the normal card. + if (!cancelled) setMode("telegram"); + }); + + return () => { + cancelled = true; + }; + }, [searchParams, loginLocal, navigate, redirectTo]); + return (
Condor

Condor

- {loggingIn ? ( + {loggingIn || (mode === null && !searchParams.get("token")) ? (

Signing in...

diff --git a/setup-environment.sh b/setup-environment.sh index 47e596b72..3406ee3d2 100755 --- a/setup-environment.sh +++ b/setup-environment.sh @@ -68,6 +68,33 @@ escape_env_value() { echo "$value" } +# Set (or replace) KEY=VALUE in .env, creating the file if it does not exist. +set_env_var() { + local key="$1" + local value + value="$(escape_env_value "$2")" + [ -f "$ENV_FILE" ] || touch "$ENV_FILE" + if grep -q "^${key}=" "$ENV_FILE"; then + sed -i.bak "s|^${key}=.*|${key}=${value}|" "$ENV_FILE" + rm -f "$ENV_FILE.bak" + else + echo "${key}=${value}" >> "$ENV_FILE" + fi +} + +# What local mode costs, said out loud. It is a dashboard with no login at all, +# so the only thing standing between it and the network is the loopback bind. +local_mode_warning() { + echo "" + msg_warn "Local mode has NO login — whoever reaches the dashboard controls your funds" + echo -e " • It binds ${BOLD}127.0.0.1${RESET} only: reachable from this machine, nowhere else." + echo -e " • Exposing it takes an explicit ${BOLD}WEB_HOST=0.0.0.0${RESET} in .env, which puts full" + echo -e " trading control in reach of anyone who can hit the port. Only do that behind" + echo -e " something that authenticates (Tailscale, an SSH tunnel, a reverse proxy)." + echo -e " • Dashboard: ${BOLD}http://localhost:8088${RESET}" + echo "" +} + # OSC 8 clickable hyperlink (falls back to plain URL) make_link() { local url="$1" @@ -456,9 +483,9 @@ fi echo "" -# ── Step 1: Telegram Configuration ────────────────── +# ── Step 1: Run mode (Telegram or local) ──────────── -echo -e "${BOLD}Step 1: Telegram Configuration${RESET}" +echo -e "${BOLD}Step 1: How will you use Condor?${RESET}" echo "" telegram_configured=false @@ -470,11 +497,49 @@ if [ -f "$ENV_FILE" ]; then set +a fi -if [ -n "${TELEGRAM_TOKEN:-}" ] && [ -n "${ADMIN_USER_ID:-}" ]; then +# An install from before local mode existed has a token but no CONDOR_MODE. +# That is telegram — recorded now, so the mode is always explicit on disk and +# is never inferred later from whether a token happens to be there. +if [ -z "${CONDOR_MODE:-}" ] && [ -n "${TELEGRAM_TOKEN:-}" ] && [ -n "${ADMIN_USER_ID:-}" ]; then + CONDOR_MODE="telegram" + set_env_var CONDOR_MODE "telegram" +fi + +if [ "${CONDOR_MODE:-}" = "local" ]; then + msg_ok "Local mode already configured (no Telegram)" + local_mode_warning +elif [ -n "${TELEGRAM_TOKEN:-}" ] && [ -n "${ADMIN_USER_ID:-}" ]; then msg_ok "Telegram already configured" telegram_configured=true load_tailscale_choice else + echo -e " ${BOLD}1) Telegram${RESET} — control Condor from a Telegram bot (recommended)" + echo -e " ${BOLD}2) Local${RESET} — no Telegram; the dashboard runs on this machine, no login" + echo "" + while true; do + prompt_visible "Choose [1/2]" "1" "condor_mode_choice" + case "${condor_mode_choice:-}" in + 1|2) break ;; + *) msg_warn "Enter 1 (Telegram) or 2 (Local)" ;; + esac + done + echo "" +fi + +if [ "${condor_mode_choice:-}" = "2" ]; then + # Local mode: nothing to ask. No bot, no token, no admin id to look up — + # ADMIN_USER_ID=1 is the local user every keyed thing (config.yml, chat + # defaults, preferences, memory, session keys) already understands. + CONDOR_MODE="local" + ADMIN_USER_ID="1" + USE_TAILSCALE=false + set_env_var CONDOR_MODE "local" + set_env_var ADMIN_USER_ID "1" + set_env_var WEB_URL "http://localhost:8088" + set_env_var USE_TAILSCALE "false" + msg_ok ".env configured for local mode" + local_mode_warning +elif [ "${condor_mode_choice:-}" = "1" ]; then msg_info "Create a bot: $(make_link 'https://t.me/BotFather')" msg_info "Get your ID: $(make_link 'https://t.me/userinfobot')" echo "" @@ -571,6 +636,11 @@ else } > "$ENV_FILE" fi + # The mode is written explicitly even for the default, so nothing downstream + # has to guess it from the presence of a token. + CONDOR_MODE="telegram" + set_env_var CONDOR_MODE "telegram" + msg_ok ".env created" if [ -n "$SERVER_IP" ]; then msg_ok "Web server configured: http://$SERVER_IP:8088" @@ -1080,12 +1150,18 @@ echo -e " • npm: $(command_exists npm && npm --version 2>/dev/null | echo -e " • typescript: $(command_exists tsc && tsc --version 2>/dev/null || echo 'not installed')" echo "" echo -e " ${BOLD}Configuration:${RESET}" +echo -e " • Mode: ${CONDOR_MODE:-telegram}$([ "${CONDOR_MODE:-telegram}" = "local" ] && echo ' (no Telegram, dashboard only)')" echo -e " • Telegram: $([ -n "${TELEGRAM_TOKEN:-}" ] && echo 'configured' || echo 'not set')" echo -e " • AI model: ${CONDOR_DEFAULT_AGENT:-from agents/condor/AGENT.md}" echo "" echo -e " ${BOLD}Next steps:${RESET}" echo -e " ${BOLD}make install${RESET} Install Python dependencies" echo -e " ${BOLD}make run${RESET} Run Condor locally (dev)" +if [ "${CONDOR_MODE:-telegram}" = "local" ]; then +echo "" +echo -e " Then open ${BOLD}http://localhost:8088${RESET} — you are logged in already." +echo -e " ${DIM}No login, loopback only. See 'Local mode' in the README before exposing it.${RESET}" +fi if [ "${hb_api_deployed:-}" = true ]; then echo "" echo -e " Hummingbot API is running — config at ${BOLD}../hummingbot-api/.env${RESET}" From 1bd487ba4b38354c717c0749d7769fa8d784e14e Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 21 Aug 2026 13:58:17 +0300 Subject: [PATCH 55/57] (fix) local mode logs in as ADMIN_USER_ID, and a broken mode fails at boot MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Ralph's report on #214: an install configured for Telegram that flips CONDOR_MODE=local boots fine, auto-logs-in, and 500s in the browser with "user 1 is not configured in config.yml — re-run `make setup`" — and the re-run then prints "already configured" and changes nothing. Four causes, all of them "two things that must agree, and nothing binding them": * Local mode had its own identity knob. ADMIN_USER_ID decides who exists in config.yml; CONDOR_LOCAL_USER_ID decided who the dashboard logs in as, and they only coincided because `make setup` happened to write 1 to both. The knob is gone: local mode logs in as ADMIN_USER_ID, defaulting to 1 for an install that never had a Telegram id. A Telegram install that switches now logs in as *you*, keeping its servers, preferences and defaults. It also answers the question the wizard could not: 1 is not an identity, it is the admin id you do not have yet. * A non-numeric ADMIN_USER_ID was swallowed, leaving an install with no admin at all — no admin panel, no approvals, no boot notification, no local login — each surfacing later as unrelated breakage. It is now a boot error naming the value. * Local mode's user is checked at boot (check_local_user), where the .env is, instead of by the login endpoint with the browser already open. * `make setup` could not repair or switch a configured install: it printed "already configured" and skipped the block, so the advice every one of those errors gives led nowhere. It now shows the current mode, offers the other one, and stops clobbering a usable ADMIN_USER_ID with 1. It also reads the mode with a parser instead of `source`, which executes .env — one value with shell metacharacters (an API key with parentheses will do) is a syntax error that silently drops every variable below it, including a hand-added CONDOR_MODE=local, which the legacy-install migration then rewrote back to telegram. `make run` now prints the pane on a startup failure instead of "run make run-fg to see the error", so these land where they are read. --- Makefile | 33 ++++++++-- README.md | 17 ++++- condor/web/routes/auth.py | 9 ++- main.py | 5 ++ setup-environment.sh | 93 +++++++++++++++++++++----- tests/test_local_mode.py | 105 ++++++++++++++++++++++++++++-- utils/config.py | 133 +++++++++++++++++++++++++++++--------- 7 files changed, 335 insertions(+), 60 deletions(-) diff --git a/Makefile b/Makefile index a9c1d85c9..d55820f27 100644 --- a/Makefile +++ b/Makefile @@ -69,17 +69,38 @@ check-stopped: exit 1; \ fi +# `remain-on-exit` keeps the pane alive after the process dies, so a startup +# failure can be read back with capture-pane instead of scrolling away with the +# session. Config errors (a bad ADMIN_USER_ID, a mode with no token, a local user +# config.yml has never heard of) exit in the first second and used to leave +# nothing behind but "run make run-fg to see the error". +# +# It is turned back off the moment the boot is confirmed: a crash three hours +# later must still take the session with it, or `make status` and `check-stopped` +# would report a dead Condor as running. run: check-stopped build-frontend @tmux new-session -d -s $(SESSION) -c "$(CURDIR)" 'uv run python main.py' - @sleep 2 - @if tmux has-session -t $(SESSION) 2>/dev/null; then \ - echo "Condor started in tmux session '$(SESSION)'."; \ - echo " make logs - attach (detach: Ctrl+B then D)"; \ - echo " make stop - stop Condor"; \ - else \ + @tmux set-option -t $(SESSION) remain-on-exit on >/dev/null 2>&1 \ + || tmux set-window-option -t $(SESSION) remain-on-exit on >/dev/null 2>&1 || true + @sleep 3 + @if ! tmux has-session -t $(SESSION) 2>/dev/null; then \ echo "Condor exited on startup. Run 'make run-fg' to see the error."; \ exit 1; \ fi + @if [ "$$(tmux list-panes -t $(SESSION) -F '#{pane_dead}' 2>/dev/null | head -1)" = "1" ]; then \ + echo "Condor exited on startup:"; \ + echo ""; \ + tmux capture-pane -p -t $(SESSION) 2>/dev/null | grep -v '^[[:space:]]*$$' | tail -20 | sed 's/^/ /'; \ + echo ""; \ + echo " make run-fg - run in the foreground for the full output"; \ + tmux kill-session -t $(SESSION) >/dev/null 2>&1 || true; \ + exit 1; \ + fi + @tmux set-option -t $(SESSION) remain-on-exit off >/dev/null 2>&1 \ + || tmux set-window-option -t $(SESSION) remain-on-exit off >/dev/null 2>&1 || true + @echo "Condor started in tmux session '$(SESSION)'." + @echo " make logs - attach (detach: Ctrl+B then D)" + @echo " make stop - stop Condor" run-fg: build-frontend uv run python main.py diff --git a/README.md b/README.md index ab7f4237e..1cef972da 100644 --- a/README.md +++ b/README.md @@ -118,7 +118,19 @@ install configured for Telegram whose `TELEGRAM_TOKEN` goes missing **exits at boot** telling you to run `make setup`. It never falls back to a login-less dashboard. -Switching later is a `make setup` re-run (or editing `CONDOR_MODE` in `.env`). +Local mode logs in as `ADMIN_USER_ID` — the same id `config.yml`, your servers, +preferences and defaults already key on. `make setup` writes `ADMIN_USER_ID=1` +for an install that never had a Telegram id; an install that has one **keeps +it**, so switching modes keeps everything you had. + +Switching later is a `make setup` re-run — it shows the current mode and offers +the other one — or editing `CONDOR_MODE` in `.env`. Your `TELEGRAM_TOKEN` is left +in place either way, so switching back is one line. + +A mode that cannot work stops at boot, in `make run`, naming the `.env` line to +fix: telegram mode with no token, an `ADMIN_USER_ID` that is not a positive +integer, or a local user who is not in `config.yml`. None of it is deferred to +the browser. ## Commands @@ -259,7 +271,8 @@ Preferences are automatically saved and persist across sessions: ```bash CONDOR_MODE=telegram # telegram (default) or local — see Local mode TELEGRAM_TOKEN=your_bot_token # Required unless CONDOR_MODE=local -ADMIN_USER_ID=123456789 # Your Telegram user id (local mode uses 1) +ADMIN_USER_ID=123456789 # Your Telegram user id. Local mode logs in + # as this same id (1 if you never had one) OPENAI_API_KEY=sk-... # Optional, for AI features OPENROUTER_API_KEY=sk-or-... # Optional, unlocks the OpenRouter LLM picker WEB_HOST=0.0.0.0 # Optional. Overrides the dashboard bind diff --git a/condor/web/routes/auth.py b/condor/web/routes/auth.py index 261ed2193..3c2a61b91 100644 --- a/condor/web/routes/auth.py +++ b/condor/web/routes/auth.py @@ -91,14 +91,19 @@ async def local_login(response: Response): response.headers["Referrer-Policy"] = "no-referrer" response.headers["Cache-Control"] = "no-store" + # Who this is was settled at boot by ``utils.config.check_local_user``, which + # refuses to start an install this would 500 on. Reaching it here means the + # user was removed or blocked while the process was up, so the message names + # the knob that decides it rather than sending anyone back to the wizard. user_id = app_config.LOCAL_USER_ID role = get_config_manager().get_user_role(user_id) if role not in (UserRole.USER, UserRole.ADMIN): raise HTTPException( status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, detail=( - f"Local mode user {user_id} is not configured in config.yml — " - "re-run `make setup`." + f"Local mode logs in as user {user_id} (ADMIN_USER_ID in .env), " + "who is no longer an approved user in config.yml. Restore that " + "user, or run `make setup` and choose Local mode." ), ) diff --git a/main.py b/main.py index cc30454e7..3cc3b9138 100644 --- a/main.py +++ b/main.py @@ -30,6 +30,7 @@ WEB_PORT, WEB_URL, ConfigError, + check_local_user, check_startup_config, ) @@ -843,8 +844,12 @@ def main() -> None: # Refuse to start on a configuration that cannot mean what it says: telegram # mode (the default) with no token used to surface as an InvalidToken # traceback from inside PTB, and must never be quietly read as "local mode". + # check_local_user() is the same idea one layer in — local mode logs in with + # no password, so who it logs in as is settled here, at boot, and not as a + # 500 from /auth/local-login with the browser already open. try: check_startup_config() + check_local_user() except ConfigError as exc: logger.error("%s", exc) raise SystemExit(1) diff --git a/setup-environment.sh b/setup-environment.sh index 3406ee3d2..f64692a1d 100755 --- a/setup-environment.sh +++ b/setup-environment.sh @@ -82,6 +82,28 @@ set_env_var() { fi } +# Read one KEY out of .env *without executing the file*. +# +# `source` runs .env as a shell script, so one value containing shell +# metacharacters (an API key with parentheses is enough) is a syntax error that +# aborts the read and silently drops every variable below that line. That is how +# a hand-added CONDOR_MODE=local could become invisible here — and then be +# rewritten to `telegram` by the legacy-install migration below. The variables +# that decide the mode are read with this instead. +read_env_var() { + local key="$1" line + [ -f "$ENV_FILE" ] || return 0 + line="$(grep -E "^[[:space:]]*(export[[:space:]]+)?${key}[[:space:]]*=" "$ENV_FILE" 2>/dev/null | tail -1)" || return 0 + [ -n "$line" ] || return 0 + line="${line#*=}" + line="$(printf '%s' "$line" | sed -E 's/^[[:space:]]+//; s/[[:space:]]+$//')" + case "$line" in + \"*\") line="${line%\"}"; line="${line#\"}" ;; + \'*\') line="${line%\'}"; line="${line#\'}" ;; + esac + printf '%s' "$line" +} + # What local mode costs, said out loud. It is a dashboard with no login at all, # so the only thing standing between it and the network is the loopback bind. local_mode_warning() { @@ -490,13 +512,23 @@ echo "" telegram_configured=false -# Source existing .env if present +# Source existing .env if present. Best-effort only: the rest of the wizard +# reads incidental values (DEPLOY_HUMMINGBOT_API, CONDOR_DEFAULT_AGENT, the +# Tailscale settings) from here, but nothing that decides the *mode* — those are +# re-read below with read_env_var, which cannot be defeated by one unquoted +# value halfway down the file. if [ -f "$ENV_FILE" ]; then set -a source "$ENV_FILE" 2>/dev/null set +a fi +if [ -f "$ENV_FILE" ]; then + CONDOR_MODE="$(read_env_var CONDOR_MODE)" + TELEGRAM_TOKEN="$(read_env_var TELEGRAM_TOKEN)" + ADMIN_USER_ID="$(read_env_var ADMIN_USER_ID)" +fi + # An install from before local mode existed has a token but no CONDOR_MODE. # That is telegram — recorded now, so the mode is always explicit on disk and # is never inferred later from whether a token happens to be there. @@ -505,36 +537,67 @@ if [ -z "${CONDOR_MODE:-}" ] && [ -n "${TELEGRAM_TOKEN:-}" ] && [ -n "${ADMIN_US set_env_var CONDOR_MODE "telegram" fi -if [ "${CONDOR_MODE:-}" = "local" ]; then - msg_ok "Local mode already configured (no Telegram)" - local_mode_warning -elif [ -n "${TELEGRAM_TOKEN:-}" ] && [ -n "${ADMIN_USER_ID:-}" ]; then - msg_ok "Telegram already configured" - telegram_configured=true - load_tailscale_choice -else +# A configured install is *offered the other mode*, never just told it is +# already configured. Switching is the one thing a re-run has to be able to do: +# the boot errors for a broken mode all say "run make setup", and that has to be +# advice that leads somewhere. +choose_mode() { echo -e " ${BOLD}1) Telegram${RESET} — control Condor from a Telegram bot (recommended)" echo -e " ${BOLD}2) Local${RESET} — no Telegram; the dashboard runs on this machine, no login" echo "" while true; do - prompt_visible "Choose [1/2]" "1" "condor_mode_choice" + prompt_visible "Choose [1/2]" "$1" "condor_mode_choice" case "${condor_mode_choice:-}" in 1|2) break ;; *) msg_warn "Enter 1 (Telegram) or 2 (Local)" ;; esac done echo "" +} + +if [ "${CONDOR_MODE:-}" = "local" ]; then + msg_ok "Currently: Local mode — dashboard only, no Telegram" + echo "" + prompt_visible "Keep local mode? [Y/n]" "Y" "keep_mode" + echo "" + if [[ "${keep_mode:-Y}" =~ ^[Nn]$ ]]; then + choose_mode "1" + else + condor_mode_choice=2 + fi +elif [ -n "${TELEGRAM_TOKEN:-}" ] && [ -n "${ADMIN_USER_ID:-}" ]; then + msg_ok "Currently: Telegram (bot configured, admin ${ADMIN_USER_ID})" + echo "" + prompt_visible "Keep Telegram mode? [Y/n]" "Y" "keep_mode" + echo "" + if [[ "${keep_mode:-Y}" =~ ^[Nn]$ ]]; then + choose_mode "2" + else + telegram_configured=true + load_tailscale_choice + fi +else + choose_mode "1" fi if [ "${condor_mode_choice:-}" = "2" ]; then - # Local mode: nothing to ask. No bot, no token, no admin id to look up — - # ADMIN_USER_ID=1 is the local user every keyed thing (config.yml, chat - # defaults, preferences, memory, session keys) already understands. + # Local mode: nothing to ask. No bot, no token — the dashboard logs in as + # ADMIN_USER_ID, the same id config.yml, chat defaults, preferences, memory + # and session keys already key on. + # + # An existing numeric id is KEPT, never overwritten with 1: a Telegram + # install switching to local keeps its own admin, so the dashboard logs in + # as you with every server, preference and default intact. 1 is only the + # answer for an install that never had a Telegram id. CONDOR_MODE="local" - ADMIN_USER_ID="1" USE_TAILSCALE=false + if [[ "${ADMIN_USER_ID:-}" =~ ^[1-9][0-9]*$ ]]; then + msg_info "Local mode will log in as user ${ADMIN_USER_ID} (ADMIN_USER_ID in .env)" + else + ADMIN_USER_ID="1" + set_env_var ADMIN_USER_ID "1" + fi set_env_var CONDOR_MODE "local" - set_env_var ADMIN_USER_ID "1" set_env_var WEB_URL "http://localhost:8088" set_env_var USE_TAILSCALE "false" msg_ok ".env configured for local mode" diff --git a/tests/test_local_mode.py b/tests/test_local_mode.py index 8fa2cf409..ed1ffefb4 100644 --- a/tests/test_local_mode.py +++ b/tests/test_local_mode.py @@ -13,6 +13,12 @@ which mode it is in), the identity ``make setup`` writes actually resolves, and PTB still runs a job queue on an Application that was never initialized. +The identity tests carry a second lesson. Local mode logs in as +``ADMIN_USER_ID`` and nothing else: a dedicated ``CONDOR_LOCAL_USER_ID`` was a +second knob for one identity, and the two disagreed the moment a Telegram +install flipped ``CONDOR_MODE=local``. Whether that identity resolves is now +settled at boot, where the .env is, rather than by a 500 from the browser. + Sync tests driving coroutines with ``asyncio.run``: ``pytest-asyncio`` is a dev dependency but is not installed in this venv (same convention as ``tests/test_notifications.py``). @@ -29,7 +35,9 @@ from utils import config as app_config from utils.config import ( ConfigError, + check_local_user, check_startup_config, + resolve_admin_id, resolve_local_user_id, resolve_mode, resolve_web_host, @@ -155,19 +163,106 @@ def test_uvicorn_binds_the_resolved_host(monkeypatch): "env, expected", [ ({}, 1), - ({"CONDOR_LOCAL_USER_ID": ""}, 1), - ({"CONDOR_LOCAL_USER_ID": "42"}, 42), + ({"ADMIN_USER_ID": ""}, 1), + ({"ADMIN_USER_ID": " "}, 1), + # The admin *is* the local user. There is no second knob to disagree + # with this one. + ({"ADMIN_USER_ID": "42"}, 42), + ({"ADMIN_USER_ID": " 123456789 "}, 123456789), # Junk and falsy ids fall back to 1: several call sites read a falsy # user or chat id as "absent", so 0 would silently address nobody. - ({"CONDOR_LOCAL_USER_ID": "nope"}, 1), - ({"CONDOR_LOCAL_USER_ID": "0"}, 1), - ({"CONDOR_LOCAL_USER_ID": "-5"}, 1), + # check_startup_config refuses to boot on these separately. + ({"ADMIN_USER_ID": "nope"}, 1), + ({"ADMIN_USER_ID": "0"}, 1), + ({"ADMIN_USER_ID": "-5"}, 1), ], ) def test_resolve_local_user_id(env, expected): assert resolve_local_user_id(env) == expected +def test_local_user_is_the_admin_when_a_telegram_install_flips_mode(): + """The regression this collapse exists for. + + A Telegram install that sets ``CONDOR_MODE=local`` keeps its Telegram + ``ADMIN_USER_ID``. Local mode used to log in as a hardcoded ``1`` that + ``config.yml`` had never heard of, so ``/auth/local-login`` 500'd on an + install that was, by every other measure, correctly configured. + """ + env = {"CONDOR_MODE": "local", "ADMIN_USER_ID": "123456789"} + + assert resolve_local_user_id(env) == 123456789 + assert resolve_local_user_id(env) == resolve_admin_id(env) + + +@pytest.mark.parametrize("raw", ["ralphsohandsome", "1.5", "0", "-5", "1,2"]) +def test_an_unusable_admin_id_refuses_to_start(raw): + """A typo'd ADMIN_USER_ID used to be swallowed, leaving *no admin at all*. + + No admin panel, no approvals, no boot notification, and in local mode no + user to log in as — each of which surfaces later as unrelated breakage. + """ + with pytest.raises(ConfigError) as excinfo: + check_startup_config({"ADMIN_USER_ID": raw, "TELEGRAM_TOKEN": "123:abc"}) + + assert "ADMIN_USER_ID" in str(excinfo.value) + assert raw in str(excinfo.value) + + +@pytest.mark.parametrize("raw", ["1", "123456789", " 42 "]) +def test_a_usable_admin_id_starts(raw): + check_startup_config({"ADMIN_USER_ID": raw, "TELEGRAM_TOKEN": "123:abc"}) + + +# ── Local mode's user is checked at boot, not at login ── + + +def test_local_mode_refuses_to_start_on_an_unconfigured_user(): + """Ralph's report: the failure has to land in ``make run``, not the browser. + + A dashboard that boots, opens, auto-logs-in and *then* 500s tells you + nothing about the .env line that caused it. + """ + with pytest.raises(ConfigError) as excinfo: + check_local_user({"CONDOR_MODE": "local"}, get_role=lambda _uid: None) + + message = str(excinfo.value) + assert "ADMIN_USER_ID" in message + assert "make setup" in message + + +def test_local_mode_names_the_admin_id_it_actually_tried(): + """The message has to name the id, or it sends you looking in config.yml.""" + env = {"CONDOR_MODE": "local", "ADMIN_USER_ID": "999"} + + with pytest.raises(ConfigError) as excinfo: + check_local_user(env, get_role=lambda _uid: None) + + assert "999" in str(excinfo.value) + + +@pytest.mark.parametrize("role", [UserRole.ADMIN, UserRole.USER]) +def test_local_mode_starts_when_the_user_resolves(role): + check_local_user({"CONDOR_MODE": "local"}, get_role=lambda _uid: role) + + +@pytest.mark.parametrize("role", [UserRole.PENDING, UserRole.BLOCKED, None]) +def test_local_mode_refuses_a_user_who_cannot_use_the_dashboard(role): + """Same bar as ``get_current_user``: pending and blocked are not sessions.""" + with pytest.raises(ConfigError): + check_local_user({"CONDOR_MODE": "local"}, get_role=lambda _uid: role) + + +@pytest.mark.parametrize("env", [{}, {"CONDOR_MODE": "telegram"}, {"CONDOR_MODE": ""}]) +def test_the_local_user_check_is_a_no_op_in_telegram_mode(env): + """Telegram mode has its own auth; this check must never touch it.""" + + def _explode(_uid): # pragma: no cover - must not be called + raise AssertionError("config.yml consulted outside local mode") + + check_local_user(env, get_role=_explode) + + def test_setup_template_gives_the_local_user_a_working_identity(tmp_path, monkeypatch): """What ``make setup`` writes in local mode has to actually resolve. diff --git a/utils/config.py b/utils/config.py index c43fd57da..ca4d85919 100644 --- a/utils/config.py +++ b/utils/config.py @@ -1,19 +1,39 @@ import os +from typing import Optional from urllib.parse import urlparse from dotenv import load_dotenv load_dotenv() -# Primary admin user ID - this user has full control over the bot -# Set via ADMIN_USER_ID environment variable -ADMIN_USER_ID = None -_admin_id_str = os.environ.get("ADMIN_USER_ID", "").strip() -if _admin_id_str: + +class ConfigError(RuntimeError): + """The process is configured in a way it must not start with.""" + + +def resolve_admin_id(env=None) -> Optional[int]: + """The primary admin's user id, or ``None`` when it is unset or unusable. + + Junk resolves to ``None`` rather than raising here, because this runs at + import time and every module reads this file; the refusal happens once, at + boot, in :func:`check_startup_config`. Non-positive ids count as unusable: + several call sites read a falsy user or chat id as "absent", so ``0`` would + silently address nobody. + """ + env = os.environ if env is None else env + raw = (env.get("ADMIN_USER_ID") or "").strip() + if not raw: + return None try: - ADMIN_USER_ID = int(_admin_id_str) + value = int(raw) except ValueError: - pass + return None + return value if value > 0 else None + + +# Primary admin user ID - this user has full control over the bot +# Set via ADMIN_USER_ID environment variable +ADMIN_USER_ID = resolve_admin_id() TELEGRAM_TOKEN = os.environ.get("TELEGRAM_TOKEN") @@ -50,10 +70,6 @@ # unauthenticated dashboard — local mode has no authentication at all. -class ConfigError(RuntimeError): - """The process is configured in a way it must not start with.""" - - MODE_TELEGRAM = "telegram" MODE_LOCAL = "local" @@ -88,35 +104,48 @@ def resolve_web_host(env=None) -> str: def resolve_local_user_id(env=None) -> int: - """The user id local mode logs in as. ``1`` unless told otherwise. + """The user id local mode logs in as: the admin, or ``1`` when there is none. - Local mode deliberately introduces no new identity concept: ``config.yml``'s + Local mode deliberately introduces no new identity concept — not even its + own. It logs in as ``ADMIN_USER_ID``, the same id ``config.yml``'s ``admin_id``, ``server_access.owner_id``, ``chat_defaults``, preferences, - memory and session keys all key on an integer user id today and keep working - untouched. ``1`` and not ``0`` because several call sites test a user or chat - id for truthiness, where a falsy id reads as "absent". + memory and session keys already key on, and the one ``ConfigManager`` + materialises a user record for. Two knobs for one identity is what broke: + a Telegram install that flipped ``CONDOR_MODE=local`` kept its Telegram + ``ADMIN_USER_ID`` while the dashboard tried to log in as a hardcoded ``1`` + that ``config.yml`` had never heard of. + + ``1`` remains the answer for an install that never had a Telegram id — it is + what ``make setup`` writes as ``ADMIN_USER_ID`` in local mode — and ``1`` + rather than ``0`` because several call sites read a falsy id as "absent". """ - env = os.environ if env is None else env - raw = (env.get("CONDOR_LOCAL_USER_ID") or "").strip() - if raw: - try: - value = int(raw) - if value > 0: - return value - except ValueError: - pass - return 1 + return resolve_admin_id(env) or 1 def check_startup_config(env=None) -> None: """Refuse to start on a configuration that cannot mean what it says. - The one case that matters: telegram mode (including the default, i.e. an - unset ``CONDOR_MODE``) with no token. Today that surfaces deep inside PTB as - an ``InvalidToken`` traceback; more importantly it must *never* be treated - as "well, local mode then". + Two cases, both of which used to surface far from their cause: + + * Telegram mode (including the default, i.e. an unset ``CONDOR_MODE``) with + no token. That surfaced deep inside PTB as an ``InvalidToken`` traceback; + more importantly it must *never* be treated as "well, local mode then". + * An ``ADMIN_USER_ID`` that is not a positive integer. That used to be + swallowed silently, leaving an install with **no admin at all**: no admin + panel, no approvals, no boot notification, and in local mode no user to + log in as — all of it looking like unrelated breakage later. """ env = os.environ if env is None else env + + raw_admin = (env.get("ADMIN_USER_ID") or "").strip() + if raw_admin and resolve_admin_id(env) is None: + raise ConfigError( + f"ADMIN_USER_ID is not a valid user id: {raw_admin!r}.\n" + "It must be a positive integer — your numeric Telegram user id " + "(get it from https://t.me/userinfobot), or 1 in local mode.\n" + "Fix it in .env, or run `make setup`." + ) + if ( resolve_mode(env) != MODE_LOCAL and not (env.get("TELEGRAM_TOKEN") or "").strip() @@ -128,6 +157,50 @@ def check_startup_config(env=None) -> None: ) +def check_local_user(env=None, get_role=None) -> None: + """In local mode, refuse to start unless the user it logs in as exists. + + Local mode logs in without a password, so the *only* thing that decides who + that is happens before any request: ``ADMIN_USER_ID`` must resolve to a + ``config.yml`` user with a role that can use the dashboard. When it does + not, the failure belongs here — at boot, in ``make run``, next to the file + you have to edit — and not as a 500 from ``/auth/local-login`` once the + browser is already open. + + ``get_role`` is injectable so this is testable without a config file; it + defaults to the real :class:`ConfigManager`, whose load also materialises + the admin's user record from the environment. + """ + env = os.environ if env is None else env + if resolve_mode(env) != MODE_LOCAL: + return + + user_id = resolve_local_user_id(env) + if get_role is None: + from config_manager import get_config_manager + + get_role = get_config_manager().get_user_role + + from config_manager import UserRole + + if get_role(user_id) not in (UserRole.USER, UserRole.ADMIN): + raw_admin = (env.get("ADMIN_USER_ID") or "").strip() + source = ( + f"ADMIN_USER_ID={raw_admin} in .env" + if raw_admin + else "the local-mode default (ADMIN_USER_ID is not set in .env)" + ) + raise ConfigError( + f"Local mode logs in as user {user_id} — {source} — but that user " + f"is not an approved user in config.yml.\n" + "Run `make setup` and choose Local mode, or set ADMIN_USER_ID in " + ".env to a user that exists in config.yml.\n" + "Switching an existing Telegram install to local mode: keep your " + "Telegram ADMIN_USER_ID and the dashboard logs in as you, with all " + "your servers and preferences intact." + ) + + CONDOR_MODE = resolve_mode() LOCAL_MODE = CONDOR_MODE == MODE_LOCAL WEB_HOST = resolve_web_host() From ed2eeb74f38ca237f94a1eb3f7bc79138ef8360e Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 21 Aug 2026 14:37:41 +0300 Subject: [PATCH 56/57] (test) add the tests for default-server sync, DEX upstream status and position attribution These cover work already on this branch but were never staged: the config.yml -first default server (and the pinned-server case the inversion must not break), the GeckoTerminal refusal counters and the 503-vs-404 the pool lookup answers when it was throttled, and the executor_ids a held position carries so a pool page can tell its own hold from another pool's. --- tests/test_default_server_sync.py | 216 ++++++++++++++++++++++++ tests/test_dex_upstream_status.py | 155 +++++++++++++++++ tests/test_position_pool_attribution.py | 56 ++++++ 3 files changed, 427 insertions(+) create mode 100644 tests/test_default_server_sync.py create mode 100644 tests/test_dex_upstream_status.py create mode 100644 tests/test_position_pool_attribution.py diff --git a/tests/test_default_server_sync.py b/tests/test_default_server_sync.py new file mode 100644 index 000000000..55785d9ad --- /dev/null +++ b/tests/test_default_server_sync.py @@ -0,0 +1,216 @@ +"""A default server picked on one surface is the default on the other. + +Both surfaces write ``chat_defaults`` in config.yml when the user picks a +default. Telegram *also* writes the choice into its pickle ``user_data``, and +``get_effective_server`` used to read that pickle first — so a default set in the +web dashboard was invisible to a live Telegram session: config.yml said one +server, the in-memory pickle said the one chosen before it, and the pickle won +every lookup until the bot process restarted. Logging out and back in on the web +made the web agree with itself; nothing made Telegram agree with the web. + +So config.yml is read first now, and the pickle is kept as a cache of it. The one +thing that must survive the inversion is a *pinned* server: a routine or agent run +launched against a named server must not be re-resolved to the chat's default. +""" + +import asyncio + +import pytest + +import config_manager as cm_module +from condor.preferences import SERVER_PIN_KEY, USER_PREFERENCES_KEY, set_active_server +from condor.routine_store import WebRoutineContext + +CHAT_ID = 4242 +WEB_PICK = "brigado_2" +TELEGRAM_PICK = "local" + + +class FakeConfigManager: + """Only the state ``get_effective_server`` reads, plus the preference writes.""" + + def __init__(self): + self._data = { + "servers": {WEB_PICK: {}, TELEGRAM_PICK: {}}, + "chat_defaults": {}, + } + self.preference_writes: list[tuple[int, str]] = [] + + def set_user_preference(self, user_id, key, value): + self.preference_writes.append((user_id, key)) + + def get_user_preferences(self, user_id): + return {} + + +@pytest.fixture +def cm(monkeypatch): + fake = FakeConfigManager() + monkeypatch.setattr(cm_module, "get_config_manager", lambda: fake) + return fake + + +def _telegram_user_data(server: str) -> dict: + """What a live Telegram session holds after the user picked ``server``.""" + user_data: dict = {} + set_active_server(user_data, server) + return user_data + + +def test_a_default_set_on_the_web_reaches_a_live_telegram_session(cm): + user_data = _telegram_user_data(TELEGRAM_PICK) + + # The web dashboard's "set default": config.yml only, no access to the pickle. + cm._data["chat_defaults"][CHAT_ID] = WEB_PICK + + assert cm_module.get_effective_server(CHAT_ID, user_data) == WEB_PICK + + +def test_the_pickle_is_corrected_rather_than_left_disagreeing(cm): + user_data = _telegram_user_data(TELEGRAM_PICK) + cm._data["chat_defaults"][CHAT_ID] = WEB_PICK + + cm_module.get_effective_server(CHAT_ID, user_data) + + assert user_data[USER_PREFERENCES_KEY]["general"]["active_server"] == WEB_PICK + + +def test_agreeing_with_config_costs_no_write(cm): + # This runs on every server lookup, and the setter persists the whole + # preference section — so a pickle that already agrees must stay silent. + user_data = _telegram_user_data(WEB_PICK) + user_data["_user_id"] = 7 + cm._data["chat_defaults"][CHAT_ID] = WEB_PICK + cm.preference_writes.clear() + + cm_module.get_effective_server(CHAT_ID, user_data) + + assert cm.preference_writes == [] + + +def test_a_chat_with_no_default_still_uses_the_users_own_pick(cm): + user_data = _telegram_user_data(TELEGRAM_PICK) + + assert cm_module.get_effective_server(CHAT_ID, user_data) == TELEGRAM_PICK + + +def test_a_deleted_server_named_in_config_does_not_win(cm): + user_data = _telegram_user_data(TELEGRAM_PICK) + cm._data["chat_defaults"][CHAT_ID] = "deleted" + + assert cm_module.get_effective_server(CHAT_ID, user_data) == TELEGRAM_PICK + + +def test_a_run_pinned_to_a_server_is_never_re_resolved(cm): + # A routine launched against WEB_PICK from a chat whose default is + # TELEGRAM_PICK: the launch server is the answer, or the run files its + # results under a server it never touched. + cm._data["chat_defaults"][CHAT_ID] = TELEGRAM_PICK + ctx = WebRoutineContext(WEB_PICK, bot=object(), chat_id=CHAT_ID) + + assert ctx.user_data[SERVER_PIN_KEY] is True + assert cm_module.get_effective_server(CHAT_ID, ctx.user_data) == WEB_PICK + + +def test_nothing_configured_resolves_to_nothing(cm): + assert cm_module.get_effective_server(CHAT_ID, {}) is None + assert cm_module.get_effective_server(CHAT_ID, None) is None + + +# --------------------------------------------------------------------------- +# The gate that keeps the pickle honest for handlers that never resolve. +# +# Most handlers (/portfolio, /bots, the gateway screens) read the pickle copy +# via ``get_active_server`` instead of resolving through +# ``get_effective_server``. Nothing corrected that copy on their path, so a +# default set on the web stayed invisible to /portfolio until the user happened +# to open /servers, which resolves as a side effect. ``@restricted`` runs on +# every Telegram update, so the correction belongs there. +# --------------------------------------------------------------------------- + + +class _FakeChat: + def __init__(self, chat_id): + self.id = chat_id + + +class _FakeUser: + id = 7 + username = "fede" + + +class _FakeUpdate: + def __init__(self, chat_id): + self.effective_chat = _FakeChat(chat_id) + self.effective_user = _FakeUser() + self.message = None + self.callback_query = None + + +class _FakeContext: + def __init__(self, user_data): + self.user_data = user_data + + +@pytest.fixture +def auth(monkeypatch, cm): + """``utils.auth`` with the config manager it imported swapped for the fake.""" + import utils.auth as auth_module + from config_manager import UserRole + + cm.get_user_role = lambda user_id: UserRole.USER + monkeypatch.setattr(auth_module, "get_config_manager", lambda: cm) + return auth_module + + +def test_a_handler_reading_the_pickle_sees_the_new_default_on_the_next_command( + auth, cm +): + """What /portfolio does: read ``get_active_server``, never resolve.""" + from condor.preferences import get_active_server + + user_data = _telegram_user_data(TELEGRAM_PICK) + seen = [] + + @auth.restricted + async def portfolio_like(update, context): + seen.append(get_active_server(context.user_data)) + + cm._data["chat_defaults"][CHAT_ID] = WEB_PICK # the web dashboard's pick + asyncio.run(portfolio_like(_FakeUpdate(CHAT_ID), _FakeContext(user_data))) + + assert seen == [WEB_PICK] + + +def test_a_pinned_run_is_not_rewritten_by_the_gate(auth, cm): + from condor.preferences import get_active_server + + cm._data["chat_defaults"][CHAT_ID] = TELEGRAM_PICK + ctx = WebRoutineContext(WEB_PICK, bot=object(), chat_id=CHAT_ID) + + @auth.restricted + async def handler(update, context): + return None + + asyncio.run(handler(_FakeUpdate(CHAT_ID), ctx)) + + assert get_active_server(ctx.user_data) == WEB_PICK + + +def test_the_refresh_never_blocks_the_command(auth, cm, monkeypatch): + import config_manager as broken + + monkeypatch.setattr( + broken, + "get_config_manager", + lambda: (_ for _ in ()).throw(RuntimeError("config unreadable")), + ) + ran = [] + + @auth.restricted + async def handler(update, context): + ran.append(True) + + asyncio.run(handler(_FakeUpdate(CHAT_ID), _FakeContext({}))) + + assert ran == [True] diff --git a/tests/test_dex_upstream_status.py b/tests/test_dex_upstream_status.py new file mode 100644 index 000000000..16e4af172 --- /dev/null +++ b/tests/test_dex_upstream_status.py @@ -0,0 +1,155 @@ +"""The DEX panel is told when GeckoTerminal is throttling Condor. + +Everything the panel draws — pool rows, TVL, a pool page, its chart — comes from +one per-IP GeckoTerminal budget, and every failure behind it degrades to an empty +list. Without a state to read, "this chain has no pools" and "the budget is spent, +ask again in ten seconds" arrive at the browser looking identical. + +Two shapes are pinned here. The refusal *counters*, because a spent per-minute +budget refuses without opening the breaker — ``rate_limited`` stays False while +the rows are still missing, so only a counter delta proves it happened. And the +pool lookup answering 503 rather than 404 when it was refused: one of those is +worth retrying and the other never resolves. +""" + +import asyncio + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.dex as dex_routes +from condor import pool_data +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +POOL = "8sLbNZoA1cfnvMJLPfp98ZLAnFSYCFApfJKMbiXNLwxj" +USER = WebUser(id=222, username="u", first_name="U", role="user") + + +class FakeConfigManager: + def has_server_access(self, user_id, name, *a, **kw): + return True + + async def get_client(self, name): + raise AssertionError("no Gateway call belongs in these paths") + + +@pytest.fixture +def client(monkeypatch): + cm = FakeConfigManager() + monkeypatch.setattr(dex_routes, "get_config_manager", lambda: cm) + monkeypatch.setattr("condor.web.auth.get_config_manager", lambda: cm) + app = FastAPI() + app.include_router(dex_routes.router) + app.dependency_overrides[get_current_user] = lambda: USER + return TestClient(app) + + +@pytest.fixture(autouse=True) +def _clean_budget(): + pool_data.reset_gecko_throttle() + yield + pool_data.reset_gecko_throttle() + + +def test_a_healthy_budget_reports_itself_as_not_limited(client): + body = client.get("/servers/srv/dex/upstream").json() + + assert body["rate_limited"] is False + assert body["retry_in"] == 0 + assert body["budget"] == pool_data._GECKO_RATE_LIMIT + + +def test_an_open_breaker_reports_a_cooldown_to_count_down(client): + pool_data._trip_breaker() + + body = client.get("/servers/srv/dex/upstream").json() + + assert body["rate_limited"] is True + assert 0 < body["retry_in"] <= pool_data._GECKO_COOLDOWN + assert body["count_429"] == 1 + + +def test_the_counters_move_when_a_request_is_refused_without_a_429(client): + """A spent budget refuses silently — the counter is the only evidence.""" + before = client.get("/servers/srv/dex/upstream").json() + + # Spend the minute, then ask for one more slot than it holds. + async def spend(): + for _ in range(pool_data._GECKO_RATE_LIMIT): + await pool_data._acquire_rate_slot() + with pytest.raises(pool_data.GeckoRateLimited): + await pool_data._acquire_rate_slot() + + asyncio.run(spend()) + + after = client.get("/servers/srv/dex/upstream").json() + assert after["rate_limited"] is False, "a spent budget must not open the breaker" + assert after["throttled_calls"] > before["throttled_calls"] + assert after["requests_last_minute"] == pool_data._GECKO_RATE_LIMIT + + +def test_an_empty_page_says_whether_the_fetch_was_refused(client, monkeypatch): + async def refused(*a, **kw): + pool_data._trip_breaker() + return {"pools": [], "has_more": False} + + monkeypatch.setattr(pool_data, "list_gecko_pools_page", refused) + + body = client.get("/servers/srv/dex/pools?source=gecko").json() + + assert body["pools"] == [] + assert body["upstream"]["throttled_request"] is True + assert body["upstream"]["rate_limited"] is True + + +def test_a_page_that_simply_has_no_rows_is_not_blamed_on_the_budget( + client, monkeypatch +): + async def empty(*a, **kw): + return {"pools": [], "has_more": False} + + monkeypatch.setattr(pool_data, "list_gecko_pools_page", empty) + + body = client.get("/servers/srv/dex/pools?source=gecko").json() + + assert body["upstream"]["throttled_request"] is False + assert body["upstream"]["rate_limited"] is False + + +def test_a_refused_pool_lookup_is_a_503_the_browser_can_retry(client, monkeypatch): + async def refused(*a, **kw): + pool_data._trip_breaker() + return None + + monkeypatch.setattr(pool_data, "fetch_pool_by_address", refused) + + res = client.get(f"/servers/srv/dex/pools/{POOL}") + + assert res.status_code == 503 + assert "rate limit" in res.json()["detail"].lower() + + +def test_an_unknown_pool_is_still_a_404(client, monkeypatch): + async def missing(*a, **kw): + return None + + monkeypatch.setattr(pool_data, "fetch_pool_by_address", missing) + + res = client.get(f"/servers/srv/dex/pools/{POOL}") + + assert res.status_code == 404 + + +def test_favourites_carry_the_same_state_as_the_browser(client, monkeypatch): + async def refused(*a, **kw): + pool_data._trip_breaker() + return [] + + monkeypatch.setattr(pool_data, "fetch_pools_by_addresses", refused) + + body = client.get(f"/servers/srv/dex/pools-by-address?addresses={POOL}").json() + + assert body["pools"] == [] + assert body["upstream"]["throttled_request"] is True diff --git a/tests/test_position_pool_attribution.py b/tests/test_position_pool_attribution.py new file mode 100644 index 000000000..756717495 --- /dev/null +++ b/tests/test_position_pool_attribution.py @@ -0,0 +1,56 @@ +"""A held position carries the executors it came from, so a pool page can place it. + +On a DEX pool page the connector *is* the network, so connector + trading pair +puts one `solana-mainnet-beta` SOL-USDC hold on every Solana SOL-USDC pool — +Orca's page and Meteora's alike — and each reads as if the position were its own. + +The API's own attribution is `executor_ids`, and an LP executor records the pool +it opened in. Passing the ids through is what lets the pool workspace tell a hold +that belongs to the pool on screen from one held in another pool of the same pair +(dropped), and from a hold no executor ties to any pool at all — a swap, whose +route the aggregator picked, which stays on the chart labelled "any pool". +""" + +from condor.web.routes.positions import _normalize_position + + +def test_the_executors_behind_a_position_survive_normalization(): + row = _normalize_position( + { + "trading_pair": "SOL-USDC", + "connector_name": "solana-mainnet-beta", + "position_side": "LONG", + "net_amount_base": 0.115990048, + "buy_breakeven_price": 86.4017516622, + "executor_count": 3, + "executor_ids": ["exec-a", "exec-b", "exec-c"], + }, + "executor", + "master_account", + ) + + assert row["executor_ids"] == ["exec-a", "exec-b", "exec-c"] + assert row["executor_count"] == 3 + + +def test_a_position_without_executors_normalizes_to_an_empty_list(): + # Bot positions come from a controller's `positions_summary`, which carries + # no ids. An absent list must not become `None`: the browser reads it as + # "nothing ties this to a pool", not as a missing field to guess about. + row = _normalize_position( + {"trading_pair": "BTC-USDT", "connector_name": "binance"}, + "bot", + "bot/main", + ) + + assert row["executor_ids"] == [] + + +def test_an_upstream_that_answers_something_else_is_not_trusted(): + row = _normalize_position( + {"trading_pair": "BTC-USDT", "executor_ids": "exec-a"}, + "executor", + "master_account", + ) + + assert row["executor_ids"] == [] From 3bb324f804c3474ace5cf33aa23b28165ba2202d Mon Sep 17 00:00:00 2001 From: cardosofede Date: Fri, 21 Aug 2026 17:02:33 +0300 Subject: [PATCH 57/57] (feat) count every install and ask for telemetry consent in the dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two halves of the same hole. `install` was only ever emitted from the Telegram consent callback, so an install whose admin never tapped the prompt was never counted at all — and a local-mode install, which has no bot to be prompted through, could not be counted even in principle. Its consent stayed `unknown` forever, pinned to the ping floor with no way to say otherwise. Counting now happens in `telemetry.init()`, at the ping floor, without an answer, guarded by `consent.mark_install_reported()` so it stays exactly once per install whichever surface gets there first. That makes the claim the module docstring already made — "every install is counted, from the first boot, with no answer required" — true rather than aspirational. Asking now happens in the dashboard too: a one-question banner in the shell for an install that has never answered, and a permanent Settings → Privacy card that every seat can read and only the admin can change. Both render the same `DISCLOSURE` the Telegram message is built from, served by `GET /settings/telemetry`, because a privacy claim written down twice is one that will eventually disagree with itself. The Telegram copy is byte-identical to before. The envelope also carries `config.mode` now (`telegram` | `local`), which is how many installs run without Telegram — the question local mode exists to answer, and previously invisible. --- PRIVACY.md | 9 +- condor/telemetry/__init__.py | 16 ++- condor/telemetry/consent.py | 22 +++ condor/telemetry/context.py | 16 +++ condor/telemetry/prompt.py | 92 +++++++++--- condor/web/routes/settings.py | 10 ++ .../src/components/TelemetryConsentBanner.tsx | 72 ++++++++++ frontend/src/components/layout/AppShell.tsx | 6 + .../components/settings/TelemetrySettings.tsx | 125 ++++++++++++++++ frontend/src/hooks/useTelemetry.ts | 46 ++++++ frontend/src/lib/api.ts | 49 +++++++ frontend/src/pages/Settings.tsx | 3 + main.py | 7 +- tests/test_telemetry.py | 83 +++++++++++ tests/test_telemetry_consent_web.py | 136 ++++++++++++++++++ 15 files changed, 660 insertions(+), 32 deletions(-) create mode 100644 frontend/src/components/TelemetryConsentBanner.tsx create mode 100644 frontend/src/components/settings/TelemetrySettings.tsx create mode 100644 frontend/src/hooks/useTelemetry.ts create mode 100644 tests/test_telemetry_consent_web.py diff --git a/PRIVACY.md b/PRIVACY.md index 31a17d4d1..a2fb1cb2f 100644 --- a/PRIVACY.md +++ b/PRIVACY.md @@ -37,6 +37,7 @@ Every batch carries one context block describing the *deployment*, not you: | `app.version`, `app.branch` | `54ad4dc`, `main` | Which commit is actually running, so we know what to support. | | `app.python`, `app.os`, `app.arch`, `app.in_docker` | `3.12`, `linux`, `arm64`, `true` | What to test against. | | `config.*` | `has_gateway: true`, `user_count: 3`, `server_count: 2`, `llm_providers: ["openai"]` | Counts and capability flags. Numbers and fixed provider *names* — never a server name, never a URL, never a key. | +| `config.mode` | `telegram` or `local` | Whether the install runs Telegram at all, or only the dashboard. One of two fixed names. | | `dropped` | `0` | How many events the rate limiter discarded. An honest count, so a quiet incident is visibly quiet. | And the events themselves: @@ -136,9 +137,11 @@ Three ways to control it, in order of precedence: 1. **Environment** — `CONDOR_TELEMETRY=off` in your `.env`. The one full kill switch: it overrides everything, suppresses the prompt, and is the right answer for an install that must send nothing at all. -2. **The dashboard API** — `PUT /api/v1/settings/telemetry?level=ping|usage` - (admin only; `off` is rejected here). `GET` the same path to see the - current state. +2. **The dashboard** — Settings → Privacy, which every seat can read and only + the admin can change (`GET`/`PUT /api/v1/settings/telemetry?level=ping|usage`; + `off` is rejected there). This is also where an install that runs without + Telegram is asked in the first place, since it has no bot to be asked + through. 3. **`config.yml`** — edit the `telemetry` section directly: ```yaml diff --git a/condor/telemetry/__init__.py b/condor/telemetry/__init__.py index 1ba66862c..8f7bc13a8 100644 --- a/condor/telemetry/__init__.py +++ b/condor/telemetry/__init__.py @@ -62,11 +62,17 @@ def init(hosted: bool = True) -> str: ``hosted`` marks a process that owns a flush job, so events buffer in memory instead of going straight to a spool file. The MCP subprocess passes False. - Priming does two things and only two: it resolves the level once so + Priming does three things and only three: it resolves the level once so :func:`emit` never has to read the disk on a hot path, and — only if the - level is not ``off`` — it materializes the install's random ids. With the - ping floor that happens on the very first boot; only an install silenced by - ``CONDOR_TELEMETRY=off`` is left exactly as it was found. + level is not ``off`` — it materializes the install's random ids and counts + the install. With the ping floor all three happen on the very first boot; + only an install silenced by ``CONDOR_TELEMETRY=off`` is left exactly as it + was found. + + Counting here, rather than when the admin answers the prompt, is what makes + "every install is counted" true rather than aspirational: an install whose + admin ignores the prompt — and a local-mode install, which has no prompt to + ignore — is still an install. ``mark_install_reported`` keeps it to once. """ from condor.telemetry import consent, emitter @@ -74,6 +80,8 @@ def init(hosted: bool = True) -> str: effective = consent.refresh() if effective != consent.OFF: consent.ensure_identity() + if consent.mark_install_reported(): + emit("install") return effective diff --git a/condor/telemetry/consent.py b/condor/telemetry/consent.py index 8b27db13b..0ce169ed4 100644 --- a/condor/telemetry/consent.py +++ b/condor/telemetry/consent.py @@ -217,6 +217,28 @@ def set_level(new_level: str) -> str: return new_level +# ── Install counting ───────────────────────────────────────────────────── + + +def mark_install_reported() -> bool: + """True the first time this install is counted, False forever after. + + The ``install`` event carries no properties — it *is* the count — so it has + to be emitted exactly once per install, by whichever surface reaches it + first: boot, the Telegram prompt, or the dashboard card. + + It used to be emitted only from the Telegram consent callback, which meant + an install that never answered was never counted at all — and a local-mode + install, which has no prompt to answer, could not be counted even in + principle. ``ping`` is the floor, so counting needs no answer; this flag is + what keeps it from being counted twice. + """ + if _section().get("install_reported"): + return False + _update(install_reported=True) + return True + + # ── First-use tracking ─────────────────────────────────────────────────── diff --git a/condor/telemetry/context.py b/condor/telemetry/context.py index e5202e826..e12c4c360 100644 --- a/condor/telemetry/context.py +++ b/condor/telemetry/context.py @@ -95,9 +95,25 @@ def _llm_providers() -> list[str]: return found +def _mode() -> str: + """``telegram`` | ``local``. Which surface this install actually runs. + + One of two fixed names, never a token, a chat id or a host — it answers + "how many installs run Condor without Telegram at all?", which is the whole + point of local mode and was previously invisible. + """ + try: + from utils.config import CONDOR_MODE + + return CONDOR_MODE + except Exception: + return "unknown" + + def config_shape() -> dict: """Counts and capability flags. Every field is a bool, an int, or a fixed name.""" shape: dict = { + "mode": _mode(), "has_web": True, "has_gateway": False, "has_hb_api": False, diff --git a/condor/telemetry/prompt.py b/condor/telemetry/prompt.py index 8d9ec4613..c37529204 100644 --- a/condor/telemetry/prompt.py +++ b/condor/telemetry/prompt.py @@ -5,6 +5,12 @@ is sent at most once per version, the intent is written to disk *before* the message goes out (a crash loop must not re-ask forever), and until it is answered the install stays at the ``ping`` floor — counted, nothing more. + +Telegram is not the only surface that asks. A local-mode install has no bot to +message, so the dashboard asks instead — ``GET /api/v1/settings/telemetry`` +serves :data:`DISCLOSURE` to the consent card. Both surfaces render the same +copy from the same constant: a privacy claim written down twice is a privacy +claim that will eventually disagree with itself. """ from __future__ import annotations @@ -15,19 +21,61 @@ CALLBACK_PREFIX = "telemetry" -_TEXT = ( - "Help improve Condor?\n\n" - "Condor counts installs so the project knows it is used: a random id, the " - "version, and an uptime ping — nothing about you or your trading. That is " - "always on.\n\n" - "It can also send an anonymous, allowlisted usage summary: which commands " - "and screens get used, what breaks, and which models agents run. That part " - "is up to you.\n\n" - "Never included: API keys, wallet addresses, server names or URLs, " - "trading pairs, amounts, balances, positions, prompts or agent replies, " - "and no Telegram id or username.\n\n" - "Full details in PRIVACY.md at the root of the repo, which also says how " - "to change this answer at any time." +# The two answers, in the order both surfaces offer them. `off` is deliberately +# absent — install counting is the floor (see ``consent.ANSWER_LEVELS``). +OPTIONS = ( + {"level": "usage", "label": "Yes, share usage summaries"}, + {"level": "ping", "label": "Only count my install"}, +) + +# Everything an install is told before it answers. +DISCLOSURE = { + "headline": "Help improve Condor?", + "always_on": ( + "Condor counts installs so the project knows it is used: a random id, " + "the version, and an uptime ping — nothing about you or your trading. " + "That is always on." + ), + "optional": ( + "It can also send an anonymous, allowlisted usage summary: which " + "commands and screens get used, what breaks, and which models agents " + "run. That part is up to you." + ), + # A list in the browser, one sentence in Telegram. The last entry is phrased + # to close the sentence :func:`_never_line` builds. + "never": [ + "API keys", + "wallet addresses", + "server names or URLs", + "trading pairs", + "amounts", + "balances", + "positions", + "prompts or agent replies", + "Telegram id or username", + ], + "doc": ( + "Full details in PRIVACY.md at the root of the repo, which also says " + "how to change this answer at any time." + ), + "options": [dict(option) for option in OPTIONS], +} + + +def _never_line() -> str: + """The ``never`` list as the one sentence Telegram has always sent.""" + items = DISCLOSURE["never"] + return f"Never included: {', '.join(items[:-1])}, and no {items[-1]}." + + +_TEXT = "\n\n".join( + ( + DISCLOSURE["headline"], + DISCLOSURE["always_on"], + DISCLOSURE["optional"], + _never_line(), + DISCLOSURE["doc"], + ) ) @@ -38,15 +86,11 @@ def keyboard(): [ [ InlineKeyboardButton( - "Yes, share usage summaries", - callback_data=f"{CALLBACK_PREFIX}:usage", - ) - ], - [ - InlineKeyboardButton( - "Only count my install", callback_data=f"{CALLBACK_PREFIX}:ping" + option["label"], + callback_data=f"{CALLBACK_PREFIX}:{option['level']}", ) - ], + ] + for option in OPTIONS ] ) @@ -106,7 +150,11 @@ async def callback_handler(update, context) -> None: chosen = consent.grant(answer) from condor.telemetry import emitter - emitter.emit("install") + # Almost always a no-op: boot counted this install already. It stays + # here for the install whose first boot was silenced by + # `CONDOR_TELEMETRY=off` and which is answering after that was lifted. + if consent.mark_install_reported(): + emitter.emit("install") if chosen == consent.PING: await query.edit_message_text( "Thanks. Condor will only report that this install exists and " diff --git a/condor/web/routes/settings.py b/condor/web/routes/settings.py index df1bb2411..e97ba4fb9 100644 --- a/condor/web/routes/settings.py +++ b/condor/web/routes/settings.py @@ -894,9 +894,17 @@ async def get_telemetry_settings(user: WebUser = Depends(get_current_user)): The collector address is fixed in the source, so consent is the only thing that decides whether anything is transmitted: at level ``off`` no event is recorded in the first place. + + Readable by every seat on purpose — anyone using an install should be able + to see what it shares — while ``can_change`` reports who may actually + answer. That is the admin, because consent is install-wide, and it is also + the discriminator the dashboard consent card uses, so asking costs no + second request. ``disclosure`` is the same copy the Telegram prompt sends. """ from condor.telemetry import consent, emitter, outbox + from condor.telemetry.prompt import DISCLOSURE + cm = get_config_manager() return { "consent": consent.state(), "level": consent.level(), @@ -904,6 +912,8 @@ async def get_telemetry_settings(user: WebUser = Depends(get_current_user)): "endpoint_configured": bool(outbox.endpoint()), "pending_events": emitter.buffered(), "privacy_doc": "PRIVACY.md", + "can_change": cm.is_admin(user.id), + "disclosure": DISCLOSURE, } diff --git a/frontend/src/components/TelemetryConsentBanner.tsx b/frontend/src/components/TelemetryConsentBanner.tsx new file mode 100644 index 000000000..acc45132c --- /dev/null +++ b/frontend/src/components/TelemetryConsentBanner.tsx @@ -0,0 +1,72 @@ +import { Loader2, ShieldCheck } from "lucide-react"; +import { Link } from "react-router-dom"; + +import { shouldAskConsent, useSetTelemetryLevel, useTelemetry } from "@/hooks/useTelemetry"; +import type { TelemetryLevel } from "@/lib/api"; + +/** + * The dashboard's half of the consent prompt. + * + * A Telegram install gets asked once, next to the boot notification, because + * that is the one moment the admin is already looking. A local-mode install has + * no bot to be asked through, so the equivalent moment is the first dashboard + * it opens — which is this strip. + * + * There is no dismiss, because both buttons are answers and one click retires + * it forever; a "later" would only add a way to be asked again. It renders for + * nobody else: not for a non-admin seat, not once answered on either surface, + * and not when `CONDOR_TELEMETRY` has already decided. + */ +export function TelemetryConsentBanner() { + const { data } = useTelemetry(); + const mutation = useSetTelemetryLevel(); + + if (!shouldAskConsent(data) || !data) return null; + + const { disclosure } = data; + + return ( +
+ + + {/* `min-w-0` so the copy wraps inside its own column instead of pushing + the buttons onto a row of their own. */} +

+ {disclosure.headline}{" "} + {disclosure.optional}{" "} + + What is never sent + +

+ +
+ {mutation.isPending && ( + + )} + {disclosure.options.map((option, index) => ( + + ))} +
+ + {mutation.isError && ( +

+ Could not save that. {(mutation.error as Error).message} +

+ )} +
+ ); +} diff --git a/frontend/src/components/layout/AppShell.tsx b/frontend/src/components/layout/AppShell.tsx index 3ebc2bae9..8a43fefc6 100644 --- a/frontend/src/components/layout/AppShell.tsx +++ b/frontend/src/components/layout/AppShell.tsx @@ -18,6 +18,7 @@ import { NavLink, Outlet, useLocation, useNavigate } from "react-router-dom"; import { ConnectKeysOverlay } from "@/components/ConnectKeysOverlay"; import { ErrorBoundary } from "@/components/ErrorBoundary"; import { ReportIssueDialog } from "@/components/ReportIssueDialog"; +import { TelemetryConsentBanner } from "@/components/TelemetryConsentBanner"; import { ChatProvider } from "@/hooks/useChat"; import { useCredentials } from "@/hooks/useCredentials"; import { usePrefetchData } from "@/hooks/usePrefetchData"; @@ -175,6 +176,11 @@ function AppShellBody() {
+ {/* Asks once, for an install that has never answered (FEAT-023). Renders + nothing for everyone else, including on the chat workspace, which owns + its own scrolling — hence outside `main` rather than inside it. */} + + {/* Main content */}
+ +
+ ); + } + + const { disclosure, level, consent, env_overridden, can_change } = data; + const locked = env_overridden || !can_change; + // At `unknown` no option is the answer yet, so none is shown as chosen — + // the install is at the ping floor, but nobody has said so. + const chosen = consent === "granted" ? level : null; + + return ( +
+
+

+ + {disclosure.headline} +

+
+

{disclosure.always_on}

+

{disclosure.optional}

+
+
+ +
+

+ Never sent +

+
    + {disclosure.never.map((item) => ( +
  • + {item} +
    • + ))} +
+
+ +
+

+ This install +

+
+ {disclosure.options.map((option) => { + const active = chosen === option.level; + return ( + + ); + })} +
+ + {env_overridden && ( +

+ + + CONDOR_TELEMETRY is set in this install's environment + and overrides anything chosen here. Effective level:{" "} + {level}. + +

+ )} + {!env_overridden && !can_change && ( +

+ + + Telemetry is install-wide, so only the admin can change it. + Effective level: {level}. + +

+ )} + {mutation.isError && ( +

+ Could not save that. {(mutation.error as Error).message} +

+ )} +
+ +

{disclosure.doc}

+
+ ); +} diff --git a/frontend/src/hooks/useTelemetry.ts b/frontend/src/hooks/useTelemetry.ts new file mode 100644 index 000000000..2405ddec5 --- /dev/null +++ b/frontend/src/hooks/useTelemetry.ts @@ -0,0 +1,46 @@ +import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query"; + +import { api } from "@/lib/api"; +import type { TelemetryLevel, TelemetrySettingsResponse } from "@/lib/api"; + +export const TELEMETRY_KEY = ["telemetry-settings"] as const; + +/** + * The install's telemetry state, shared by the two places that render it: the + * consent banner in the shell and the card in Settings → Privacy. One query key, + * so opening Settings right after answering the banner costs no second request + * and cannot show a staler answer than the one just given. + */ +export function useTelemetry() { + return useQuery({ + queryKey: TELEMETRY_KEY, + queryFn: api.getTelemetrySettings, + // Nothing else in the app changes this, and it survives restarts on disk. + staleTime: 5 * 60 * 1000, + retry: false, + }); +} + +export function useSetTelemetryLevel() { + const qc = useQueryClient(); + return useMutation({ + mutationFn: (level: TelemetryLevel) => api.setTelemetryLevel(level), + onSuccess: (res) => { + // Write the answer through rather than refetching: the banner unmounts on + // `consent !== "unknown"`, and a refetch round-trip would leave it on + // screen long enough to look like the click did nothing. + qc.setQueryData(TELEMETRY_KEY, (old: TelemetrySettingsResponse | undefined) => + old ? { ...old, level: res.level, consent: "granted" as const } : old, + ); + }, + }); +} + +/** + * Should we ask? Only an install that has never answered, only the admin who is + * allowed to answer, and never when the environment has pinned the level — in + * that case there is no answer left to give. + */ +export function shouldAskConsent(data: TelemetrySettingsResponse | undefined): boolean { + return !!data && data.consent === "unknown" && data.can_change && !data.env_overridden; +} diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts index 5c82a6a0a..dddca9c41 100644 --- a/frontend/src/lib/api.ts +++ b/frontend/src/lib/api.ts @@ -1000,6 +1000,43 @@ export interface VoiceSettingsResponse { available_languages: Record; } +// ── Telemetry (FEAT-023) ── + +/** The only two answers. `off` is reachable only from the environment. */ +export type TelemetryLevel = "ping" | "usage"; + +export interface TelemetryOption { + level: TelemetryLevel; + label: string; +} + +/** + * What the install is told before it answers, served by the backend so this + * copy and the Telegram prompt's cannot drift (`condor/telemetry/prompt.py`). + */ +export interface TelemetryDisclosure { + headline: string; + always_on: string; + optional: string; + never: string[]; + doc: string; + options: TelemetryOption[]; +} + +export interface TelemetrySettingsResponse { + /** `unknown` until someone answers — that is what the consent card asks. */ + consent: "unknown" | "granted" | "denied"; + level: "off" | TelemetryLevel; + /** `CONDOR_TELEMETRY` is pinned in the environment; nothing here can change it. */ + env_overridden: boolean; + endpoint_configured: boolean; + pending_events: number; + privacy_doc: string; + /** Consent is install-wide, so only the admin may answer. */ + can_change: boolean; + disclosure: TelemetryDisclosure; +} + // ── Chat ── export interface ChatAgentOption { @@ -2142,6 +2179,18 @@ export const api = { body: JSON.stringify(data), }), + // ── Telemetry (FEAT-023) ── + + getTelemetrySettings: () => + apiFetch("/api/v1/settings/telemetry"), + + /** Admin only (403 otherwise), and 409 when `CONDOR_TELEMETRY` is pinned. */ + setTelemetryLevel: (level: TelemetryLevel) => + apiFetch<{ level: TelemetryLevel; consent: string }>( + `/api/v1/settings/telemetry?level=${level}`, + { method: "PUT" }, + ), + // ── Chat ── getOpenRouterModels: () => diff --git a/frontend/src/pages/Settings.tsx b/frontend/src/pages/Settings.tsx index 325384ead..9e2b21113 100644 --- a/frontend/src/pages/Settings.tsx +++ b/frontend/src/pages/Settings.tsx @@ -7,6 +7,7 @@ import { ApiKeysSettings } from "@/components/settings/ApiKeysSettings"; import { CustomProvidersSettings } from "@/components/settings/CustomProvidersSettings"; import { GatewaySettings } from "@/components/settings/GatewaySettings"; import { ServersSettings } from "@/components/settings/ServersSettings"; +import { TelemetrySettings } from "@/components/settings/TelemetrySettings"; import { VoiceSettings } from "@/components/settings/VoiceSettings"; import { ADMIN_USERS_KEY, adminApi } from "@/lib/admin-api"; import { useAuth } from "@/lib/auth"; @@ -17,6 +18,7 @@ const TABS = [ { key: "keys", label: "Keys and Wallets" }, { key: "llm", label: "LLM Endpoints" }, { key: "voice", label: "Voice & AI" }, + { key: "privacy", label: "Privacy" }, ] as const; /** Admin-only, appended to TABS when the user turns out to be an admin (ARCH-177). */ @@ -80,6 +82,7 @@ export function Settings() { {tab === "keys" && } {tab === "llm" && } {tab === "voice" && } + {tab === "privacy" && } {tab === "admin" && }
); diff --git a/main.py b/main.py index 3cc3b9138..22d6a609e 100644 --- a/main.py +++ b/main.py @@ -988,9 +988,10 @@ async def _run_dual(application: Application) -> None: # Ask, once, whether this install wants to be counted (FEAT-023). Sent # next to the boot notification because that is the one moment the admin # is already looking. Until it is answered, nothing is collected. The - # prompt is a Telegram message with inline buttons, so local mode has - # nowhere to ask and stays silent — which leaves consent `unknown`, - # which emits nothing. + # prompt is a Telegram message with inline buttons, so local mode is + # skipped here and asked in the dashboard instead (the consent card in + # Settings → Privacy). Either way the install is already counted: + # `telemetry.init()` does that at the ping floor, without an answer. if not LOCAL_MODE: from condor.telemetry.prompt import maybe_prompt_admin diff --git a/tests/test_telemetry.py b/tests/test_telemetry.py index 32a016f40..6294af3c8 100644 --- a/tests/test_telemetry.py +++ b/tests/test_telemetry.py @@ -425,6 +425,89 @@ def test_the_install_context_carries_no_identity(install): assert identifying not in blob +def test_the_envelope_says_which_surface_the_install_runs(install, monkeypatch): + """`mode` is one of two fixed names — the adoption signal local mode was + built for, and nothing that could point at a host or a chat.""" + monkeypatch.setattr("utils.config.CONDOR_MODE", "local", raising=False) + assert context.config_shape()["mode"] == "local" + monkeypatch.setattr("utils.config.CONDOR_MODE", "telegram", raising=False) + assert context.config_shape()["mode"] == "telegram" + + +# ── Every install is counted ───────────────────────────────────────────── + + +def test_a_fresh_install_is_counted_without_anyone_answering(install): + """The claim that gives this feature its floor. `install` used to be emitted + only from the Telegram consent callback, so an install that never answered + — every local-mode one — was never counted at all.""" + import condor.telemetry as telemetry + + telemetry.init(hosted=True) + + assert consent.state() == consent.UNKNOWN # nobody answered anything + assert [e["name"] for e in emitter.drain()[0]] == ["install"] + + +def test_an_install_is_counted_once_however_many_times_it_boots(install): + import condor.telemetry as telemetry + + telemetry.init(hosted=True) + emitter.drain() + + telemetry.init(hosted=True) + telemetry.init(hosted=True) + assert emitter.drain()[0] == [] + + +def test_answering_the_prompt_does_not_count_the_install_a_second_time(install): + import condor.telemetry as telemetry + + telemetry.init(hosted=True) + emitter.drain() + + consent.grant("usage") + assert consent.mark_install_reported() is False + + +def test_an_install_silenced_by_the_environment_is_never_counted(install, monkeypatch): + """`CONDOR_TELEMETRY=off` is the kill switch, and counting is not exempt + from it — nothing is recorded and config.yml is left as it was found.""" + monkeypatch.setattr("utils.config.CONDOR_TELEMETRY", "off", raising=False) + import condor.telemetry as telemetry + + assert telemetry.init(hosted=True) == consent.OFF + assert emitter.drain()[0] == [] + assert not (install / "config.yml").exists() + + +# ── Asking, on both surfaces ───────────────────────────────────────────── + + +def test_telegram_and_the_dashboard_disclose_exactly_the_same_thing(): + """Two prompts mean two chances to disagree about what is collected. The + dashboard card renders `DISCLOSURE`; the Telegram message is built from it.""" + from condor.telemetry.prompt import _TEXT, DISCLOSURE + + assert DISCLOSURE["headline"] in _TEXT + assert DISCLOSURE["always_on"] in _TEXT + assert DISCLOSURE["optional"] in _TEXT + assert DISCLOSURE["doc"] in _TEXT + for never in DISCLOSURE["never"]: + assert never in _TEXT + + +def test_the_prompt_offers_no_way_to_switch_counting_off(): + """`off` is the operator's env kill switch, never a button — on either + surface. A stray option here would quietly make the floor optional.""" + from condor.telemetry.prompt import DISCLOSURE, OPTIONS + + levels = {option["level"] for option in OPTIONS} + assert levels == set(consent.ANSWER_LEVELS) + assert consent.OFF not in levels + assert [o["level"] for o in DISCLOSURE["options"]] == [o["level"] for o in OPTIONS] + + # ── Out of process ─────────────────────────────────────────────────────── diff --git a/tests/test_telemetry_consent_web.py b/tests/test_telemetry_consent_web.py new file mode 100644 index 000000000..b57880216 --- /dev/null +++ b/tests/test_telemetry_consent_web.py @@ -0,0 +1,136 @@ +"""The dashboard half of the telemetry consent prompt (FEAT-023). + +A Telegram install is asked once, next to the boot notification. An install +running without Telegram has no bot to be asked through, so before this the +question could not be put to it at all: consent stayed ``unknown`` forever and +no amount of willingness could move it off the ping floor. These tests pin the +contract the dashboard consent card is built on. +""" + +import pytest +from fastapi import FastAPI +from starlette.testclient import TestClient + +import condor.web.routes.settings as settings_routes +from condor.telemetry import consent +from condor.web.auth import get_current_user +from condor.web.models import WebUser + +ADMIN = WebUser(id=1, username="admin", first_name="A", role="admin") +TRADER = WebUser(id=2, username="trader", first_name="T", role="user") + + +class FakeConfigManager: + def is_admin(self, user_id): + return user_id == ADMIN.id + + +@pytest.fixture +def app(tmp_path, monkeypatch): + """An isolated install, so the answers these tests give are written to a + config.yml that dies with the test.""" + import config_manager as cm_module + from condor.telemetry import emitter + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr("utils.config.CONDOR_TELEMETRY", None, raising=False) + cm_module.ConfigManager.reset_instance() + emitter.discard_buffer() + consent.refresh() + + # Only the route's own authorization check is faked; `consent` keeps its + # real ConfigManager so a PUT genuinely lands on disk. + monkeypatch.setattr(settings_routes, "get_config_manager", FakeConfigManager) + + api = FastAPI() + api.include_router(settings_routes.router) + yield api + + cm_module.ConfigManager.reset_instance() + emitter.discard_buffer() + consent.refresh() + + +def as_user(app, user): + app.dependency_overrides[get_current_user] = lambda: user + return TestClient(app) + + +def test_an_install_that_never_answered_reports_itself_as_unanswered(app): + """`unknown` plus `can_change` is the whole trigger for the consent card.""" + body = as_user(app, ADMIN).get("/settings/telemetry").json() + + assert body["consent"] == "unknown" + assert body["level"] == "ping" # counted meanwhile — the floor + assert body["can_change"] is True + assert body["env_overridden"] is False + + +def test_every_seat_may_read_what_the_install_shares(app): + """Transparency is not an admin feature. Answering is.""" + body = as_user(app, TRADER).get("/settings/telemetry").json() + + assert body["consent"] == "unknown" + assert body["can_change"] is False + assert body["disclosure"]["never"] # the same list, for everyone + + +def test_the_card_is_served_the_same_words_telegram_sends(app): + from condor.telemetry.prompt import DISCLOSURE + + body = as_user(app, ADMIN).get("/settings/telemetry").json() + assert body["disclosure"] == DISCLOSURE + + +def test_a_non_admin_cannot_answer_for_the_install(app): + resp = as_user(app, TRADER).put("/settings/telemetry", params={"level": "usage"}) + + assert resp.status_code == 403 + assert consent.state() == consent.UNKNOWN + + +def test_answering_in_the_dashboard_retires_the_question(app): + """The card's whole job: move consent off `unknown` and record the level. + Once it is granted the banner stops rendering, on this surface and the + other one.""" + resp = as_user(app, ADMIN).put("/settings/telemetry", params={"level": "usage"}) + + assert resp.status_code == 200 + assert resp.json() == {"level": "usage", "consent": "granted"} + assert consent.state() == consent.GRANTED + assert consent.level() == consent.USAGE + + after = as_user(app, ADMIN).get("/settings/telemetry").json() + assert after["consent"] == "granted" + + +def test_only_counting_is_a_real_answer_too(app): + """ "Only count my install" is an answer, not a refusal to answer — it has + to retire the prompt, or the admin gets asked again forever.""" + as_user(app, ADMIN).put("/settings/telemetry", params={"level": "ping"}) + + assert consent.state() == consent.GRANTED + assert consent.level() == consent.PING + + +def test_the_dashboard_cannot_switch_install_counting_off(app): + resp = as_user(app, ADMIN).put("/settings/telemetry", params={"level": "off"}) + + assert resp.status_code == 400 + assert consent.state() == consent.UNKNOWN + + +def test_an_environment_pinned_install_is_not_asked_and_cannot_be_answered( + app, monkeypatch +): + """With `CONDOR_TELEMETRY` set there is no question left to put: the card + reports the override instead of offering buttons, and a PUT is refused.""" + monkeypatch.setattr("utils.config.CONDOR_TELEMETRY", "off", raising=False) + consent.refresh() + + body = as_user(app, ADMIN).get("/settings/telemetry").json() + assert body["env_overridden"] is True + assert body["level"] == "off" + + resp = as_user(app, ADMIN).put("/settings/telemetry", params={"level": "usage"}) + assert resp.status_code == 409