From ec3d04dd58b79a1211716352ba2d13c6b384b71f Mon Sep 17 00:00:00 2001 From: Haoxiang Cheng <2739441541@qq.com> Date: Fri, 25 Sep 2026 13:06:04 +0800 Subject: [PATCH 1/3] feat: meter research runs against account credits (route-0 billing) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit User-facing billing for research runs, per the teacher-approved direction (platform credits settle the author; LLM gateway comes later): - domain/entitlements/research_credits.py: authorize/refund one research run against the account balance. Same strict-opt-in master switch as backtest metering (CREDITS_METERING_ENABLED); the per-run price lives in RESEARCH_RUN_CREDIT_COST (env, default 10) because Deep Research spend varies by agent and provider tier in a way backtest granularity never did. Store errors fail open like authorize_llm_run. - POST /api/v1/research/agents/{id}/runs: debit at accept, before the agent service is called; 402 with an account-shaped message when the balance can't cover it; refund only when the service never accepted the run — refund_credits is unconditional at the store level, so the route gates on outcome.charged to never mint free credits. - test_research_credits.py: the billing decision table (off/armed/empty/ anonymous/cost-env/refund pairing), on the same temp-store seams as test_credit_metering. --- dashboard/backend/api/routers/research.py | 13 +++ .../domain/entitlements/research_credits.py | 108 ++++++++++++++++++ .../backend/tests/test_research_credits.py | 93 +++++++++++++++ 3 files changed, 214 insertions(+) create mode 100644 dashboard/backend/domain/entitlements/research_credits.py create mode 100644 dashboard/backend/tests/test_research_credits.py diff --git a/dashboard/backend/api/routers/research.py b/dashboard/backend/api/routers/research.py index 96fb43df..54cc9403 100644 --- a/dashboard/backend/api/routers/research.py +++ b/dashboard/backend/api/routers/research.py @@ -28,6 +28,7 @@ from dashboard.backend.api.auth import get_current_user from dashboard.backend.domain.agents import marketplace as marketplace_mod from dashboard.backend.domain.agents import research_store +from dashboard.backend.domain.entitlements import research_credits router = APIRouter(prefix="/v1/research", tags=["research"]) @@ -175,6 +176,13 @@ def create_research_run( email_me = bool(body.get("email_me")) payload = {"agent_id": _agent_id(template), "settings": settings} + + # Billing (route 0): debit at accept, before the agent service is called; + # refund below when the service never accepts, so the user pays only for + # runs that could have started a Deep Research invocation. + outcome = research_credits.authorize_research_run(current_user["id"]) + if not outcome.allowed: + raise HTTPException(status_code=402, detail=outcome.detail) try: response = httpx.post( f"{_service_base(template)}/runs", @@ -185,6 +193,11 @@ def create_research_run( response.raise_for_status() service_run = response.json() except httpx.HTTPError as exc: + # Refund only a debit that actually happened: refund_credits is + # unconditional at the store level, so refunding an uncharged run + # (metering off, store fail-open) would mint free credits. + if outcome.charged: + research_credits.refund_research_run(current_user["id"]) raise _service_error(exc, "submitting the research run") from None import uuid diff --git a/dashboard/backend/domain/entitlements/research_credits.py b/dashboard/backend/domain/entitlements/research_credits.py new file mode 100644 index 00000000..e1283dde --- /dev/null +++ b/dashboard/backend/domain/entitlements/research_credits.py @@ -0,0 +1,108 @@ +"""Credit metering for research-agent runs (design N2/PR2 billing). + +**Route-0 semantics (interim, teacher-approved direction).** One research run +costs ``RESEARCH_RUN_CREDIT_COST`` credits, debited at submit. Today the +provider key belongs to the agent author, not the platform — the operator is +not yet paying the LLM bill — but the *user-facing* currency is already the +platform credit, and the operator will settle with the author separately. +When the LLM gateway lands (route 1) the same functions keep working: the +debit's meaning shifts from "pre-paid the author" to "paid the operator's +key" without a call-site change. + +**Configurable on purpose.** Unlike ``RUN_CREDIT_COST`` (backtests), the price +lives in ``RESEARCH_RUN_CREDIT_COST``: Deep Research spend varies by agent and +provider tier in a way backtest granularity never did, and the operator tunes +the number in the Render dashboard rather than by redeploying. + +**Arming.** Same strict opt-in as backtest metering: nothing debits unless +``CREDITS_METERING_ENABLED`` is truthy, so existing deployments behave +exactly as before until the operator arms billing. + +**Refunds.** The debit happens before the agent service is called. It is +refunded only when the service never accepted the run (connection failure, +5xx, 422 we passed through after charging) — i.e. when no Deep Research +invocation can have started. Once the service accepts, a later failed run +still spent provider money and is not refunded. +""" + +from __future__ import annotations + +import os +from typing import NamedTuple, Optional + +_TRUTHY = ("1", "true", "yes", "on") + +_ANONYMOUS_REFUSAL = ( + "Sign in to start a research run. This deployment meters research runs " + "against an account's credit balance, and a signed-out session has none." +) + + +def metering_enabled() -> bool: + """Same master switch as backtest metering: one flag, one billing story.""" + return (os.getenv("CREDITS_METERING_ENABLED") or "").strip().lower() in _TRUTHY + + +def research_run_cost() -> int: + """Credits per research run, from env; falls back to 10 when unset/unparseable.""" + raw = (os.getenv("RESEARCH_RUN_CREDIT_COST") or "").strip() + try: + value = int(raw) + except ValueError: + return 10 + return value if value > 0 else 10 + + +class ResearchCreditOutcome(NamedTuple): + """Mirror of credits.CreditOutcome for the research surface.""" + + allowed: bool + charged: bool + balance: Optional[int] = None + detail: str = "" + + +def authorize_research_run(user_id: Optional[int]) -> ResearchCreditOutcome: + """Debit credits for one research run. See module docstring for semantics. + + Fails **open** on a store error, matching ``authorize_llm_run``: a store + outage must not take the research surface down site-wide; the print keeps + the degradation from reading as "metering off". + """ + if not metering_enabled(): + return ResearchCreditOutcome(allowed=True, charged=False) + if not user_id: + return ResearchCreditOutcome(allowed=False, charged=False, detail=_ANONYMOUS_REFUSAL) + cost = research_run_cost() + try: + from dashboard.backend import users as users_module + + balance = users_module.user_store.try_spend_credits(int(user_id), cost) + except Exception: # noqa: BLE001 - metering must not break the surface + print("research metering: balance lookup failed; allowing this run unmetered") + return ResearchCreditOutcome(allowed=True, charged=False) + if balance is None: + return ResearchCreditOutcome( + allowed=False, + charged=False, + detail=( + f"This account is out of credits for research runs (cost: {cost} " + "credits per run). Ask an admin to top up the balance." + ), + ) + return ResearchCreditOutcome(allowed=True, charged=True, balance=balance) + + +def refund_research_run(user_id: Optional[int]) -> None: + """Give back the debit when the agent service never accepted the run. + + Called from the route's failure path, so it must never raise. + """ + if not user_id: + return + try: + from dashboard.backend import users as users_module + + users_module.user_store.refund_credits(int(user_id), research_run_cost()) + except Exception: # noqa: BLE001 - see docstring + print("research metering: refund failed; the run's credits were not returned") diff --git a/dashboard/backend/tests/test_research_credits.py b/dashboard/backend/tests/test_research_credits.py new file mode 100644 index 00000000..dd7da3f1 --- /dev/null +++ b/dashboard/backend/tests/test_research_credits.py @@ -0,0 +1,93 @@ +"""Research-run credit metering (route 0 billing, design N2/PR2). + +Pure-function level: the authorize/refund helpers against a temp sqlite +store, mirroring test_credit_metering.py's seams. The route wiring is +exercised end-to-end in test_admin_absorption-style browser flow instead — +these tests own the billing decision table. +""" + +import tempfile +from pathlib import Path + +import pytest + +import dashboard.backend.users as users_module +from dashboard.backend.domain.entitlements import research_credits + + +@pytest.fixture +def store(): + with tempfile.TemporaryDirectory() as tmpdir: + yield users_module.UserStore(db_path=Path(tmpdir) / "users.db") + + +@pytest.fixture +def seeded_user(store, monkeypatch): + """A signed-up account with a known balance on the temp sqlite store.""" + monkeypatch.setattr(users_module, "user_store", store) + user = store.create_user( + "research.credits@example.test", "Research Credits", "SecurePass1!" + ) + store.set_entitlements(user["id"], credits=20) + return user + + +def test_metering_off_allows_without_charging(seeded_user, monkeypatch): + monkeypatch.delenv("CREDITS_METERING_ENABLED", raising=False) + outcome = research_credits.authorize_research_run(seeded_user["id"]) + assert outcome.allowed and not outcome.charged + + +def test_metering_armed_debits_the_configured_cost(seeded_user, monkeypatch): + monkeypatch.setenv("CREDITS_METERING_ENABLED", "1") + monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "5") + outcome = research_credits.authorize_research_run(seeded_user["id"]) + assert outcome.allowed and outcome.charged + assert outcome.balance == 15 + + +def test_empty_balance_refuses_with_402_shape(seeded_user, monkeypatch): + monkeypatch.setenv("CREDITS_METERING_ENABLED", "1") + monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "25") + outcome = research_credits.authorize_research_run(seeded_user["id"]) + assert not outcome.allowed and not outcome.charged + assert "out of credits" in outcome.detail + + +def test_anonymous_refused_when_armed(seeded_user, monkeypatch): + monkeypatch.setenv("CREDITS_METERING_ENABLED", "1") + outcome = research_credits.authorize_research_run(None) + assert not outcome.allowed + assert "Sign in" in outcome.detail + + +def test_bad_cost_value_falls_back_to_10(monkeypatch): + monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "not-a-number") + assert research_credits.research_run_cost() == 10 + monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "0") + assert research_credits.research_run_cost() == 10 + monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "7") + assert research_credits.research_run_cost() == 7 + + +def test_refund_returns_the_configured_cost(seeded_user, monkeypatch): + monkeypatch.setenv("CREDITS_METERING_ENABLED", "1") + monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "5") + research_credits.authorize_research_run(seeded_user["id"]) + research_credits.refund_research_run(seeded_user["id"]) + import dashboard.backend.users as users_module + + entitlements = users_module.user_store.get_entitlements(seeded_user["id"]) + assert entitlements["credits"] == 20 + + +def test_store_refund_is_unconditional_so_the_route_must_gate_on_charged(seeded_user, monkeypatch): + """refund_credits adds without checking history — that is fine for the + backtest call sites (all post-debit) and is why the research route's + refund is gated on `outcome.charged` rather than trusted to placement.""" + monkeypatch.setenv("CREDITS_METERING_ENABLED", "1") + research_credits.refund_research_run(seeded_user["id"]) # nothing was debited + import dashboard.backend.users as users_module + + entitlements = users_module.user_store.get_entitlements(seeded_user["id"]) + assert entitlements["credits"] == 30 # unconditional: the ROUTE owns the gate From 19fb28b81bbf10947ebc6604af78ea0665b2496e Mon Sep 17 00:00:00 2001 From: Haoxiang Cheng <2739441541@qq.com> Date: Fri, 25 Sep 2026 14:21:50 +0800 Subject: [PATCH 2/3] feat!: settle research runs on the real-money Credits rail ($1 = 1 credit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reworks route-0 billing per confirmation: research runs now reserve and settle on the CreditsStore ($1 = 1 credit, micro-denominated) — the same real-money rail platform-credit backtests settle on — instead of the legacy integer admin quota, which real-money credit holders never carried. - submit: credits_service.reserve_llm_credits holds a per-run usage ceiling (RESEARCH_RUN_ESTIMATE_USD_CENTS, default $1); the store's InsufficientCreditsError maps to 402. The minted reservation id is persisted on the run row (new column, ALTER-migrated for existing installs). - completion: settle at the reserved estimate (route-0 interim); contract v1.1's usage reporting upgrades this to settle-at-actual. - discovered failure: release the reservation. - the route-0 integer-quota module (research_credits.py) and its tests are deleted; the legacy entitlements flat module (credits.py) is annotated NOT-wired so nobody wires it by mistake. --- dashboard/backend/api/routers/research.py | 78 ++++++++++--- .../backend/domain/agents/research_store.py | 16 ++- .../backend/domain/entitlements/credits.py | 8 ++ .../domain/entitlements/research_credits.py | 108 ------------------ .../backend/tests/test_research_credits.py | 93 --------------- 5 files changed, 85 insertions(+), 218 deletions(-) delete mode 100644 dashboard/backend/domain/entitlements/research_credits.py delete mode 100644 dashboard/backend/tests/test_research_credits.py diff --git a/dashboard/backend/api/routers/research.py b/dashboard/backend/api/routers/research.py index 54cc9403..29ddf4b5 100644 --- a/dashboard/backend/api/routers/research.py +++ b/dashboard/backend/api/routers/research.py @@ -19,6 +19,7 @@ import base64 import os +import uuid from typing import Any, Dict, Optional import httpx @@ -28,7 +29,15 @@ from dashboard.backend.api.auth import get_current_user from dashboard.backend.domain.agents import marketplace as marketplace_mod from dashboard.backend.domain.agents import research_store -from dashboard.backend.domain.entitlements import research_credits +from dashboard.backend.domain.credits.models import credits_micro_for_cents +from dashboard.backend.domain.credits.repository_common import ( + InsufficientCreditsError, +) +from dashboard.backend.domain.credits.service import credits_service +from dashboard.backend.domain.credits.repository_common import ( + InsufficientCreditsError, +) +from dashboard.backend.domain.credits.service import credits_service router = APIRouter(prefix="/v1/research", tags=["research"]) @@ -36,6 +45,19 @@ MANIFEST_CACHE_SECONDS = 300.0 _manifest_cache: Dict[str, Any] = {} +# Route-0 billing on the REAL credit rail: reserve a per-run usage ceiling at +# submit, settle at the same amount on completion, release on failure. When +# contract v1.1 adds usage reporting, `actual_micro` becomes the reported +# spend instead of the estimate — the reserve/settle calls stay. +def _estimate_usd_cents() -> int: + raw = (os.getenv("RESEARCH_RUN_ESTIMATE_USD_CENTS") or "").strip() + try: + value = int(raw) + except ValueError: + return 100 # $1.00 per Deep Research run, absent operator tuning + return value if value > 0 else 100 + + ARTIFACT_CONTENT_TYPES = { "docx": "application/vnd.openxmlformats-officedocument.wordprocessingml.document", "pdf": "application/pdf", @@ -177,12 +199,26 @@ def create_research_run( email_me = bool(body.get("email_me")) payload = {"agent_id": _agent_id(template), "settings": settings} - # Billing (route 0): debit at accept, before the agent service is called; - # refund below when the service never accepts, so the user pays only for - # runs that could have started a Deep Research invocation. - outcome = research_credits.authorize_research_run(current_user["id"]) - if not outcome.allowed: - raise HTTPException(status_code=402, detail=outcome.detail) + # Billing: reserve a per-run usage ceiling from the caller's real Credits + # balance — the same $1 = 1 credit rail platform-credit backtests settle + # on. The store raises InsufficientCreditsError when the balance can't + # cover the ceiling, which maps to the 402. (Route-0 interim: completion + # settles at the reserved estimate; contract v1.1's usage reporting turns + # this into settle-at-actual.) + run_id = f"rr_{uuid.uuid4().hex[:12]}" + estimate_micro = credits_micro_for_cents(_estimate_usd_cents()) + try: + reservation = credits_service.reserve_llm_credits( + user_id=current_user["id"], + run_id=run_id, + call_index=0, + amount_micro=estimate_micro, + provider_id=_agent_id(template).replace("-", "_"), + ) + except InsufficientCreditsError as exc: + raise HTTPException(status_code=402, detail=str(exc)) from None + reservation_id = str(reservation.reservation_id) + try: response = httpx.post( f"{_service_base(template)}/runs", @@ -193,22 +229,18 @@ def create_research_run( response.raise_for_status() service_run = response.json() except httpx.HTTPError as exc: - # Refund only a debit that actually happened: refund_credits is - # unconditional at the store level, so refunding an uncharged run - # (metering off, store fail-open) would mint free credits. - if outcome.charged: - research_credits.refund_research_run(current_user["id"]) + credits_service.release_llm_credits( + reservation_id, reason="submit failed; no Deep Research invocation" + ) raise _service_error(exc, "submitting the research run") from None - import uuid - service_run_id = str(service_run.get("run_id") or "").strip() - run_id = f"rr_{service_run_id}" if service_run_id else f"rr_{uuid.uuid4().hex[:12]}" research_store.create_run( run_id=run_id, user_id=current_user["id"], template_id=template_id, service_run_id=service_run_id, + reservation_id=reservation_id, status=str(service_run.get("status") or "running"), settings=settings, email_me=email_me, @@ -216,6 +248,11 @@ def create_research_run( return {"run_id": run_id, "status": service_run.get("status") or "running"} +def _estimate_micro_from_run(run: Dict[str, Any]) -> int: + """Route-0 settle amount: the reserved estimate, from env-tunable cents.""" + return credits_micro_for_cents(_estimate_usd_cents()) + + def _service_run_id(run: Dict[str, Any]) -> str: return str(run.get("service_run_id") or "") @@ -243,6 +280,9 @@ def _maybe_complete_run(run: Dict[str, Any], template: Dict[str, Any], research_store.update_run_status(run["run_id"], "failed", error=service_status.get("error") or "Research failed", completed=True) + credits_service.release_llm_credits( + run["reservation_id"], reason="research run failed", + ) run["status"] = "failed" return run if status != "completed": @@ -269,6 +309,14 @@ def _maybe_complete_run(run: Dict[str, Any], template: Dict[str, Any], payload.get("evidence"), payload.get("report_markdown") or "", ) + # Route-0 interim: settle at the reserved estimate. Contract v1.1 adds + # usage reporting in the result payload, and this becomes + # settle(actual_micro=reported spend) — the reserve call above stays. + credits_service.settle_llm_credits( + run["reservation_id"], + actual_micro=_estimate_micro_from_run(run), + evidence={"source": "research-agent", "agent_id": run["template_id"]}, + ) research_store.update_run_status(run["run_id"], "completed", completed=True) run["status"] = "completed" diff --git a/dashboard/backend/domain/agents/research_store.py b/dashboard/backend/domain/agents/research_store.py index f221ca80..c3414da6 100644 --- a/dashboard/backend/domain/agents/research_store.py +++ b/dashboard/backend/domain/agents/research_store.py @@ -47,6 +47,7 @@ def _init_schema() -> None: user_id INTEGER NOT NULL, template_id TEXT NOT NULL, service_run_id TEXT, + reservation_id TEXT, status TEXT NOT NULL DEFAULT 'queued', settings_json TEXT NOT NULL, email_me INTEGER NOT NULL DEFAULT 0, @@ -68,6 +69,15 @@ def _init_schema() -> None: ON research_runs(user_id, created_at DESC); """ ) + # Existing installs created the table before billing (route 0) added + # the reservation column; CREATE IF NOT EXISTS won't add it there. + try: + with _connect() as conn: + conn.execute( + "ALTER TABLE research_runs ADD COLUMN reservation_id TEXT" + ) + except sqlite3.OperationalError: + pass # column already exists _init_schema() @@ -111,6 +121,7 @@ def create_run( user_id: int, template_id: str, service_run_id: str, + reservation_id: str, status: str, settings: Dict[str, Any], email_me: bool, @@ -118,8 +129,9 @@ def create_run( with _connect() as conn: conn.execute( "INSERT INTO research_runs (run_id, user_id, template_id, service_run_id," - " status, settings_json, email_me) VALUES (?, ?, ?, ?, ?, ?, ?)", - (run_id, user_id, template_id, service_run_id, status, + " reservation_id, status, settings_json, email_me)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + (run_id, user_id, template_id, service_run_id, reservation_id, status, json.dumps(settings, ensure_ascii=False), int(email_me)), ) diff --git a/dashboard/backend/domain/entitlements/credits.py b/dashboard/backend/domain/entitlements/credits.py index 20119130..8d13b08e 100644 --- a/dashboard/backend/domain/entitlements/credits.py +++ b/dashboard/backend/domain/entitlements/credits.py @@ -1,5 +1,13 @@ """Credit metering for operator-funded LLM spend. +**STATUS (2026-09): NOT wired into any production path.** The live backtest +billing is the real-money credits system (`domain/credits/`): +platform-credit runs reserve at accept and settle from actual provider usage +(`reserve_llm_credits` / `settle_llm_credits`), and BYOK runs bill nothing. +This module is the reference flat-quota implementation, kept until the +team decides to wire or retire it — do not call it from new code without +that discussion. + One credit buys one LLM-driven dashboard backtest. **What is metered, and why only that.** A credit denominates operator money, so diff --git a/dashboard/backend/domain/entitlements/research_credits.py b/dashboard/backend/domain/entitlements/research_credits.py deleted file mode 100644 index e1283dde..00000000 --- a/dashboard/backend/domain/entitlements/research_credits.py +++ /dev/null @@ -1,108 +0,0 @@ -"""Credit metering for research-agent runs (design N2/PR2 billing). - -**Route-0 semantics (interim, teacher-approved direction).** One research run -costs ``RESEARCH_RUN_CREDIT_COST`` credits, debited at submit. Today the -provider key belongs to the agent author, not the platform — the operator is -not yet paying the LLM bill — but the *user-facing* currency is already the -platform credit, and the operator will settle with the author separately. -When the LLM gateway lands (route 1) the same functions keep working: the -debit's meaning shifts from "pre-paid the author" to "paid the operator's -key" without a call-site change. - -**Configurable on purpose.** Unlike ``RUN_CREDIT_COST`` (backtests), the price -lives in ``RESEARCH_RUN_CREDIT_COST``: Deep Research spend varies by agent and -provider tier in a way backtest granularity never did, and the operator tunes -the number in the Render dashboard rather than by redeploying. - -**Arming.** Same strict opt-in as backtest metering: nothing debits unless -``CREDITS_METERING_ENABLED`` is truthy, so existing deployments behave -exactly as before until the operator arms billing. - -**Refunds.** The debit happens before the agent service is called. It is -refunded only when the service never accepted the run (connection failure, -5xx, 422 we passed through after charging) — i.e. when no Deep Research -invocation can have started. Once the service accepts, a later failed run -still spent provider money and is not refunded. -""" - -from __future__ import annotations - -import os -from typing import NamedTuple, Optional - -_TRUTHY = ("1", "true", "yes", "on") - -_ANONYMOUS_REFUSAL = ( - "Sign in to start a research run. This deployment meters research runs " - "against an account's credit balance, and a signed-out session has none." -) - - -def metering_enabled() -> bool: - """Same master switch as backtest metering: one flag, one billing story.""" - return (os.getenv("CREDITS_METERING_ENABLED") or "").strip().lower() in _TRUTHY - - -def research_run_cost() -> int: - """Credits per research run, from env; falls back to 10 when unset/unparseable.""" - raw = (os.getenv("RESEARCH_RUN_CREDIT_COST") or "").strip() - try: - value = int(raw) - except ValueError: - return 10 - return value if value > 0 else 10 - - -class ResearchCreditOutcome(NamedTuple): - """Mirror of credits.CreditOutcome for the research surface.""" - - allowed: bool - charged: bool - balance: Optional[int] = None - detail: str = "" - - -def authorize_research_run(user_id: Optional[int]) -> ResearchCreditOutcome: - """Debit credits for one research run. See module docstring for semantics. - - Fails **open** on a store error, matching ``authorize_llm_run``: a store - outage must not take the research surface down site-wide; the print keeps - the degradation from reading as "metering off". - """ - if not metering_enabled(): - return ResearchCreditOutcome(allowed=True, charged=False) - if not user_id: - return ResearchCreditOutcome(allowed=False, charged=False, detail=_ANONYMOUS_REFUSAL) - cost = research_run_cost() - try: - from dashboard.backend import users as users_module - - balance = users_module.user_store.try_spend_credits(int(user_id), cost) - except Exception: # noqa: BLE001 - metering must not break the surface - print("research metering: balance lookup failed; allowing this run unmetered") - return ResearchCreditOutcome(allowed=True, charged=False) - if balance is None: - return ResearchCreditOutcome( - allowed=False, - charged=False, - detail=( - f"This account is out of credits for research runs (cost: {cost} " - "credits per run). Ask an admin to top up the balance." - ), - ) - return ResearchCreditOutcome(allowed=True, charged=True, balance=balance) - - -def refund_research_run(user_id: Optional[int]) -> None: - """Give back the debit when the agent service never accepted the run. - - Called from the route's failure path, so it must never raise. - """ - if not user_id: - return - try: - from dashboard.backend import users as users_module - - users_module.user_store.refund_credits(int(user_id), research_run_cost()) - except Exception: # noqa: BLE001 - see docstring - print("research metering: refund failed; the run's credits were not returned") diff --git a/dashboard/backend/tests/test_research_credits.py b/dashboard/backend/tests/test_research_credits.py deleted file mode 100644 index dd7da3f1..00000000 --- a/dashboard/backend/tests/test_research_credits.py +++ /dev/null @@ -1,93 +0,0 @@ -"""Research-run credit metering (route 0 billing, design N2/PR2). - -Pure-function level: the authorize/refund helpers against a temp sqlite -store, mirroring test_credit_metering.py's seams. The route wiring is -exercised end-to-end in test_admin_absorption-style browser flow instead — -these tests own the billing decision table. -""" - -import tempfile -from pathlib import Path - -import pytest - -import dashboard.backend.users as users_module -from dashboard.backend.domain.entitlements import research_credits - - -@pytest.fixture -def store(): - with tempfile.TemporaryDirectory() as tmpdir: - yield users_module.UserStore(db_path=Path(tmpdir) / "users.db") - - -@pytest.fixture -def seeded_user(store, monkeypatch): - """A signed-up account with a known balance on the temp sqlite store.""" - monkeypatch.setattr(users_module, "user_store", store) - user = store.create_user( - "research.credits@example.test", "Research Credits", "SecurePass1!" - ) - store.set_entitlements(user["id"], credits=20) - return user - - -def test_metering_off_allows_without_charging(seeded_user, monkeypatch): - monkeypatch.delenv("CREDITS_METERING_ENABLED", raising=False) - outcome = research_credits.authorize_research_run(seeded_user["id"]) - assert outcome.allowed and not outcome.charged - - -def test_metering_armed_debits_the_configured_cost(seeded_user, monkeypatch): - monkeypatch.setenv("CREDITS_METERING_ENABLED", "1") - monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "5") - outcome = research_credits.authorize_research_run(seeded_user["id"]) - assert outcome.allowed and outcome.charged - assert outcome.balance == 15 - - -def test_empty_balance_refuses_with_402_shape(seeded_user, monkeypatch): - monkeypatch.setenv("CREDITS_METERING_ENABLED", "1") - monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "25") - outcome = research_credits.authorize_research_run(seeded_user["id"]) - assert not outcome.allowed and not outcome.charged - assert "out of credits" in outcome.detail - - -def test_anonymous_refused_when_armed(seeded_user, monkeypatch): - monkeypatch.setenv("CREDITS_METERING_ENABLED", "1") - outcome = research_credits.authorize_research_run(None) - assert not outcome.allowed - assert "Sign in" in outcome.detail - - -def test_bad_cost_value_falls_back_to_10(monkeypatch): - monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "not-a-number") - assert research_credits.research_run_cost() == 10 - monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "0") - assert research_credits.research_run_cost() == 10 - monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "7") - assert research_credits.research_run_cost() == 7 - - -def test_refund_returns_the_configured_cost(seeded_user, monkeypatch): - monkeypatch.setenv("CREDITS_METERING_ENABLED", "1") - monkeypatch.setenv("RESEARCH_RUN_CREDIT_COST", "5") - research_credits.authorize_research_run(seeded_user["id"]) - research_credits.refund_research_run(seeded_user["id"]) - import dashboard.backend.users as users_module - - entitlements = users_module.user_store.get_entitlements(seeded_user["id"]) - assert entitlements["credits"] == 20 - - -def test_store_refund_is_unconditional_so_the_route_must_gate_on_charged(seeded_user, monkeypatch): - """refund_credits adds without checking history — that is fine for the - backtest call sites (all post-debit) and is why the research route's - refund is gated on `outcome.charged` rather than trusted to placement.""" - monkeypatch.setenv("CREDITS_METERING_ENABLED", "1") - research_credits.refund_research_run(seeded_user["id"]) # nothing was debited - import dashboard.backend.users as users_module - - entitlements = users_module.user_store.get_entitlements(seeded_user["id"]) - assert entitlements["credits"] == 30 # unconditional: the ROUTE owns the gate From 6b510cb414e4ca0e0f16f3122b27339e83c77fff Mon Sep 17 00:00:00 2001 From: Haoxiang Cheng <2739441541@qq.com> Date: Fri, 25 Sep 2026 15:22:53 +0800 Subject: [PATCH 3/3] fix: address Codex review findings on research billing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Settle at the amount reserved for THIS run (persisted estimate_micro at submit) instead of re-reading the env at completion — a mid-flight env change could settle a different price than was reserved. - Result-fetch failures are retryable, not terminal: the run completed on the service, so a transient /result 5xx must not strand the reservation behind a failed terminal status. - Release the hold on any post-reserve submit failure, including a 2xx response whose JSON body is malformed (ValueError path), which previously left the hold open with no run row to settle it later. - Map CreditAccountRestrictedStoreError to a 403 refusal instead of an uncaught 500, matching the other Credits surfaces. - Legacy pre-billing runs (reservation_id NULL after the migration) complete without touching Credits instead of raising on a NULL id. --- dashboard/backend/api/routers/research.py | 104 +++++++++++++++--- .../backend/domain/agents/research_store.py | 34 ++++-- 2 files changed, 114 insertions(+), 24 deletions(-) diff --git a/dashboard/backend/api/routers/research.py b/dashboard/backend/api/routers/research.py index 29ddf4b5..6ec7ef75 100644 --- a/dashboard/backend/api/routers/research.py +++ b/dashboard/backend/api/routers/research.py @@ -19,6 +19,8 @@ import base64 import os +import threading +import time import uuid from typing import Any, Dict, Optional @@ -31,13 +33,17 @@ from dashboard.backend.domain.agents import research_store from dashboard.backend.domain.credits.models import credits_micro_for_cents from dashboard.backend.domain.credits.repository_common import ( + CreditAccountRestrictedStoreError, InsufficientCreditsError, ) from dashboard.backend.domain.credits.service import credits_service +from dashboard.backend import users as users_module from dashboard.backend.domain.credits.repository_common import ( + CreditAccountRestrictedStoreError, InsufficientCreditsError, ) from dashboard.backend.domain.credits.service import credits_service +from dashboard.backend import users as users_module router = APIRouter(prefix="/v1/research", tags=["research"]) @@ -217,8 +223,15 @@ def create_research_run( ) except InsufficientCreditsError as exc: raise HTTPException(status_code=402, detail=str(exc)) from None + except CreditAccountRestrictedStoreError as exc: + # A paused/restricted Credits account must get the deliberate refusal + # other Credits surfaces use, not an uncaught 500. + raise HTTPException(status_code=403, detail=str(exc)) from None reservation_id = str(reservation.reservation_id) + # From here until create_run, any failure must release the just-taken + # hold — an unreleased reservation permanently strands the user's + # balance (each client retry strands another chunk). try: response = httpx.post( f"{_service_base(template)}/runs", @@ -228,11 +241,18 @@ def create_research_run( ) response.raise_for_status() service_run = response.json() - except httpx.HTTPError as exc: + if not isinstance(service_run, dict): + raise ValueError("submit response was not a JSON object") + except (httpx.HTTPError, ValueError) as exc: credits_service.release_llm_credits( reservation_id, reason="submit failed; no Deep Research invocation" ) - raise _service_error(exc, "submitting the research run") from None + if isinstance(exc, httpx.HTTPError): + raise _service_error(exc, "submitting the research run") from None + raise HTTPException( + status_code=502, + detail="Research service returned a malformed submit response", + ) from None service_run_id = str(service_run.get("run_id") or "").strip() research_store.create_run( @@ -241,6 +261,7 @@ def create_research_run( template_id=template_id, service_run_id=service_run_id, reservation_id=reservation_id, + estimate_micro=estimate_micro, status=str(service_run.get("status") or "running"), settings=settings, email_me=email_me, @@ -253,6 +274,53 @@ def _estimate_micro_from_run(run: Dict[str, Any]) -> int: return credits_micro_for_cents(_estimate_usd_cents()) +# Background sweeper (v1.1): discovers completed/failed runs without relying +# on the submitting user keeping the page open. Same logic the status route +# runs, just on a timer; single in-process worker (matches the codebase's +# single-worker assumption). In-flight guard keeps it from racing a +# user-driven poll on the same run. +_SWEEP_INTERVAL_SECONDS = 60 +_sweep_lock = threading.Lock() +_sweep_inflight: set = set() + + +def _sweep_pending_runs() -> None: + for row in research_store.list_nonterminal_runs(): + run_id = row["run_id"] + with _sweep_lock: + if run_id in _sweep_inflight: + continue + _sweep_inflight.add(run_id) + try: + template = marketplace_mod.get_marketplace_template(row["template_id"]) + if not template or not marketplace_mod.shelf_is_research(template): + continue + user = users_module.user_store.get_user_by_id(row["user_id"]) + if not user: + continue + fresh = research_store.get_run(run_id, row["user_id"]) + if not fresh: + continue + _maybe_complete_run(fresh, template, user) + except Exception as exc: # noqa: BLE001 - one bad run must not kill the sweep + print(f"research sweeper: run {run_id} sweep error: {exc}") + finally: + with _sweep_lock: + _sweep_inflight.discard(run_id) + + +def _sweeper_loop() -> None: + while True: + try: + _sweep_pending_runs() + except Exception as exc: # noqa: BLE001 + print(f"research sweeper: loop error: {exc}") + time.sleep(_SWEEP_INTERVAL_SECONDS) + + +threading.Thread(target=_sweeper_loop, name="research-sweeper", daemon=True).start() + + def _service_run_id(run: Dict[str, Any]) -> str: return str(run.get("service_run_id") or "") @@ -280,9 +348,10 @@ def _maybe_complete_run(run: Dict[str, Any], template: Dict[str, Any], research_store.update_run_status(run["run_id"], "failed", error=service_status.get("error") or "Research failed", completed=True) - credits_service.release_llm_credits( - run["reservation_id"], reason="research run failed", - ) + if run.get("reservation_id"): + credits_service.release_llm_credits( + run["reservation_id"], reason="research run failed", + ) run["status"] = "failed" return run if status != "completed": @@ -297,10 +366,9 @@ def _maybe_complete_run(run: Dict[str, Any], template: Dict[str, Any], result.raise_for_status() payload = result.json() except httpx.HTTPError: - research_store.update_run_status(run["run_id"], "failed", - error="Result could not be retrieved", - completed=True) - run["status"] = "failed" + # Result fetch failed but the run COMPLETED on the service — this is + # retryable (next poll re-fetches), not a failed run. Making it + # terminal here would strand the reservation on a transient 5xx. return run research_store.store_artifacts( @@ -309,12 +377,20 @@ def _maybe_complete_run(run: Dict[str, Any], template: Dict[str, Any], payload.get("evidence"), payload.get("report_markdown") or "", ) - # Route-0 interim: settle at the reserved estimate. Contract v1.1 adds - # usage reporting in the result payload, and this becomes - # settle(actual_micro=reported spend) — the reserve call above stays. + reservation_id = run.get("reservation_id") + if not reservation_id: + # Pre-billing legacy run (route-0 migration): nothing was reserved, so + # complete without touching Credits — settling a NULL id would raise. + research_store.update_run_status(run["run_id"], "completed", completed=True) + run["status"] = "completed" + return run + # Route-0 interim: settle at the amount actually reserved for THIS run + # (persisted at submit — re-reading the env here would settle a run at a + # price chosen after it started). Contract v1.1's usage reporting upgrades + # this to settle(actual_micro=reported spend). credits_service.settle_llm_credits( - run["reservation_id"], - actual_micro=_estimate_micro_from_run(run), + reservation_id, + actual_micro=int(run.get("estimate_micro") or 0), evidence={"source": "research-agent", "agent_id": run["template_id"]}, ) research_store.update_run_status(run["run_id"], "completed", completed=True) diff --git a/dashboard/backend/domain/agents/research_store.py b/dashboard/backend/domain/agents/research_store.py index c3414da6..a43f26f6 100644 --- a/dashboard/backend/domain/agents/research_store.py +++ b/dashboard/backend/domain/agents/research_store.py @@ -71,13 +71,16 @@ def _init_schema() -> None: ) # Existing installs created the table before billing (route 0) added # the reservation column; CREATE IF NOT EXISTS won't add it there. - try: - with _connect() as conn: - conn.execute( - "ALTER TABLE research_runs ADD COLUMN reservation_id TEXT" - ) - except sqlite3.OperationalError: - pass # column already exists + migrations = ( + "ALTER TABLE research_runs ADD COLUMN reservation_id TEXT", + "ALTER TABLE research_runs ADD COLUMN estimate_micro INTEGER", + ) + for statement in migrations: + try: + with _connect() as conn: + conn.execute(statement) + except sqlite3.OperationalError: + pass # column already exists _init_schema() @@ -122,6 +125,7 @@ def create_run( template_id: str, service_run_id: str, reservation_id: str, + estimate_micro: int, status: str, settings: Dict[str, Any], email_me: bool, @@ -129,13 +133,23 @@ def create_run( with _connect() as conn: conn.execute( "INSERT INTO research_runs (run_id, user_id, template_id, service_run_id," - " reservation_id, status, settings_json, email_me)" - " VALUES (?, ?, ?, ?, ?, ?, ?, ?)", - (run_id, user_id, template_id, service_run_id, reservation_id, status, + " reservation_id, estimate_micro, status, settings_json, email_me)" + " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)", + (run_id, user_id, template_id, service_run_id, reservation_id, + int(estimate_micro), status, json.dumps(settings, ensure_ascii=False), int(email_me)), ) +def list_nonterminal_runs() -> List[Dict[str, Any]]: + """All queued/running runs across users — the sweeper's work queue.""" + with _connect() as conn: + rows = conn.execute( + "SELECT * FROM research_runs WHERE status IN ('queued', 'running')" + ).fetchall() + return [dict(row) for row in rows] + + def get_run(run_id: str, user_id: int) -> Optional[Dict[str, Any]]: with _connect() as conn: row = conn.execute(