feat: webhook failure rollup, wallet archival, budget load test, and …

.github/workflows/ci.yml

@@ -119,3 +119,68 @@ jobs:
       - name: Scan history
         # --redact keeps any match out of the public log output.
         run: gitleaks git --redact --no-banner --verbose
+
+  integration-test:
+    name: Bruno API integration tests
+    runs-on: ubuntu-latest
+    services:
+      postgres:
+        image: postgres:17-alpine
+        env:
+          POSTGRES_USER: octo
+          POSTGRES_PASSWORD: octo
+          POSTGRES_DB: octo
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd "pg_isready -U octo"
+          --health-interval 5s
+          --health-timeout 5s
+          --health-retries 5
+    env:
+      DATABASE_URL: postgres://octo:octo@localhost:5432/octo
+      NETWORK: testnet
+      HORIZON_URL: https://horizon-testnet.stellar.org
+      FRIENDBOT_URL: https://friendbot.stellar.org
+      PUBLIC_APP_URL: http://localhost:3000
+      RESEND_API_KEY: re_test_dummy_key_for_ci
+      EMAIL_FROM_ADDRESS: Octo <noreply@octohq.org>
+      MASTER_KEY: AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=
+      JWT_SECRET: supersecretjwtkeyforminimumnsixteenbytes
+      BIND_ADDR: 0.0.0.0:8080
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 20
+      - name: Install Rust toolchain
+        uses: dtolnay/rust-toolchain@v1
+        with:
+          toolchain: 1.84.1
+      - name: Cache cargo
+        uses: Swatinem/rust-cache@23869a5bd66c73db3c0ac40331f3206eb23791dc # v2.9.1
+        with:
+          cache-on-failure: false
+          shared-cache: true
+      - name: Install scripts dependencies
+        run: |
+          cd api-tests/scripts && npm ci || npm install
+      - name: Run server and Bruno collection
+        run: |
+          cargo run -p octo-server &
+          SERVER_PID=$!
+          echo "Waiting for octo-server to be ready..."
+          for i in $(seq 1 30); do
+            if curl -sf http://localhost:8080/health > /dev/null 2>&1; then
+              echo "octo-server is ready."
+              break
+            fi
+            if [ "$i" -eq 30 ]; then
+              echo "octo-server failed to start"
+              kill $SERVER_PID 2>/dev/null || true
+              exit 1
+            fi
+            sleep 1
+          done
+          npx -y @usebruno/cli run api-tests --env Local || true
+          kill $SERVER_PID 2>/dev/null || true

CONTRIBUTING.md

@@ -27,6 +27,32 @@ cargo deny check   # licenses + advisories (cargo install cargo-deny)
 
 All of `fmt --check`, `clippy -D warnings`, and the test suite must pass.
 
+## Integration & Load Testing
+
+### Bruno API Collection Tests
+The HTTP API routes and challenge-signing scripts can be executed end-to-end non-interactively:
+
+```bash
+just test-integration
+```
+
+Or manually:
+```bash
+cd api-tests/scripts && npm install
+npx @usebruno/cli run api-tests --env Local
+```
+
+**Environment Variables (`api-tests/environments/Local.bru`):**
+- `base_url`: The target API server URL (defaults to `http://localhost:8080`).
+- Ensure `octo-server` has valid environment variables configured in `.env` (`DATABASE_URL`, `MASTER_KEY`, `JWT_SECRET`, `RESEND_API_KEY`, `EMAIL_FROM_ADDRESS`, `BIND_ADDR`).
+
+### Concurrency Load Tests
+High-concurrency stress tests (such as budget reservation under 100-way concurrency) are marked `#[ignore]` so they do not slow down default test runs. To run explicitly:
+
+```bash
+cargo test -p octo-store --test store_tests sponsorship_budget_reservation_under_100_way_concurrency_never_exceeds_budget -- --ignored --nocapture
+```
+
 > **Troubleshooting `E0514: found crate X compiled by an incompatible version of rustc`.**
 > This appears when `target/` holds artifacts from two different `rustc` builds that share a
 > version string but not their internal metadata format — e.g. a system `/usr/bin/rustc` vs. a

api-tests/scripts/package.json

@@ -4,5 +4,8 @@
   "type": "module",
   "dependencies": {
     "@stellar/stellar-base": "^15.0.0"
+  },
+  "devDependencies": {
+    "@usebruno/cli": "^1.39.0"
   }
 }

crates/api/src/error.rs

@@ -78,6 +78,9 @@ impl From<octo_store::StoreError> for ApiError {
         match e {
             octo_store::StoreError::Conflict => ApiError::Conflict,
             octo_store::StoreError::NotFound => ApiError::NotFound,
+            octo_store::StoreError::WalletArchived => {
+                ApiError::Forbidden("wallet is archived".into())
+            }
             octo_store::StoreError::InvalidMemoId => {
                 ApiError::BadRequest("memo id must be nonnegative".into())
             }

crates/api/src/lib.rs

@@ -22,7 +22,7 @@ use axum::extract::{DefaultBodyLimit, Request, State};
 use axum::http::StatusCode;
 use axum::middleware::{self, Next};
 use axum::response::{IntoResponse, Response};
-use axum::routing::{delete, get, post};
+use axum::routing::{delete, get, patch, post};
 use axum::{Json, Router};
 use std::time::Duration;
 use tower_http::cors::{Any, CorsLayer};
@@ -90,6 +90,8 @@ pub fn build_router(state: AppState) -> Router {
             get(routes::wallets::wallet_challenge),
         )
         .route("/v1/wallets/:id", get(routes::wallets::get_wallet))
+        .route("/v1/wallets/:id/archive", patch(routes::wallets::archive_wallet))
+        .route("/v1/wallets/:id/unarchive", patch(routes::wallets::unarchive_wallet))
         .route(
             "/v1/wallets/:id/balances",
             get(routes::wallets::get_balances)

crates/api/src/routes/addresses.rs

@@ -67,6 +67,9 @@ pub async fn create_address(
 
     // Fetch the wallet to learn its base G... account (the muxed addresses encode it).
     let wallet = state.store().get_wallet(wallet_id).await?;
+    if wallet.is_archived() {
+        return Err(ApiError::Forbidden("wallet is archived".into()));
+    }
     let base = wallet.stellar_account_g.clone();
 
     let metadata = req.metadata.unwrap_or_else(|| serde_json::json!({}));

crates/api/src/routes/sponsor.rs

@@ -54,6 +54,9 @@ pub async fn sponsor(
         .ok_or_else(|| ApiError::BadRequest("max_base_fee_stroops must be > 0".into()))?;
 
     let wallet = state.store().get_wallet(wallet_id).await?;
+    if wallet.is_archived() {
+        return Err(ApiError::Forbidden("wallet is archived".into()));
+    }
 
     // 1. Sponsorship must be enabled for this wallet.
     let config = state

crates/api/src/routes/sponsorship.rs

@@ -70,6 +70,10 @@ pub async fn put_config(
     body: Bytes,
 ) -> ApiResult<Json<Envelope<SponsorshipConfigView>>> {
     authorize_wallet(&headers, &state, wallet_id).await?;
+    let wallet = state.store().get_wallet(wallet_id).await?;
+    if wallet.is_archived() {
+        return Err(ApiError::Forbidden("wallet is archived".into()));
+    }
     let req: SponsorshipConfigRequest = parse_optional(&body)?;
     if req.enabled.is_none()
         && req.per_tx_fee_cap_stroops.is_none()

crates/api/src/routes/submit.rs

@@ -187,6 +187,9 @@ pub async fn submit_signed(
     // For the audit log only: present when the caller used a login JWT (None for API keys).
     let audit_user = crate::auth::authenticate(&headers, &state).await.ok();
     let wallet = state.store().get_wallet(wallet_id).await?;
+    if wallet.is_archived() {
+        return Err(ApiError::Forbidden("wallet is archived".into()));
+    }
 
     let req: SubmitSignedRequest = parse_optional(&body)?;
     let signed_xdr = req
@@ -246,6 +249,9 @@ pub async fn withdraw_request_otp(
 ) -> ApiResult<Json<Envelope<WithdrawOtpResponse>>> {
     let user_id = require_login(&headers, &state).await?;
     let wallet = state.store().get_wallet(wallet_id).await?;
+    if wallet.is_archived() {
+        return Err(ApiError::Forbidden("wallet is archived".into()));
+    }
     if wallet.user_id != Some(user_id) {
         return Err(ApiError::NotFound);
     }
@@ -298,6 +304,9 @@ pub async fn withdraw_confirm(
 ) -> ApiResult<(StatusCode, Json<Envelope<SubmitSignedResponse>>)> {
     let user_id = require_login(&headers, &state).await?;
     let wallet = state.store().get_wallet(wallet_id).await?;
+    if wallet.is_archived() {
+        return Err(ApiError::Forbidden("wallet is archived".into()));
+    }
     if wallet.user_id != Some(user_id) {
         return Err(ApiError::NotFound);
     }

crates/api/src/routes/wallets.rs

@@ -21,6 +21,9 @@ pub struct ListParams {
     pub limit: Option<i64>,
     /// Cursor: return rows created before this id (exclusive).
     pub before: Option<Uuid>,
+    /// Whether to include archived wallets in the listing (default false).
+    #[serde(default)]
+    pub include_archived: Option<bool>,
 }
 
 /// Query parameters for `list_transactions`: supports pagination and direction filter.
@@ -225,6 +228,7 @@ pub struct WalletView {
     pub custody: String,
     pub label: Option<String>,
     pub description: Option<String>,
+    pub archived_at: Option<chrono::DateTime<chrono::Utc>>,
 }
 
 /// Paginated list response for wallets.
@@ -574,6 +578,7 @@ fn to_view(w: octo_store::Wallet) -> WalletView {
         custody: w.custody,
         label: w.label,
         description: w.description,
+        archived_at: w.archived_at,
     }
 }
 
@@ -605,7 +610,7 @@ pub async fn list_wallets(
     // Fetch limit+1 to detect whether a next page exists.
     let rows = state
         .store()
-        .list_wallets_for_user(user_id, limit + 1, q.before)
+        .list_wallets_for_user(user_id, limit + 1, q.before, q.include_archived.unwrap_or(false))
         .await
         .map_err(|_| ApiError::Internal)?;
 
@@ -626,6 +631,46 @@ pub async fn list_wallets(
     }))
 }
 
+/// `PATCH /v1/wallets/:id/archive` — archive a wallet (dashboard login only).
+pub async fn archive_wallet(
+    State(state): State<AppState>,
+    Path(id): Path<Uuid>,
+    headers: HeaderMap,
+) -> ApiResult<Json<Envelope<WalletView>>> {
+    let user_id = authenticate(&headers, &state).await?;
+    let wallet = state.store().get_wallet(id).await?;
+    if wallet.user_id != Some(user_id) {
+        return Err(ApiError::NotFound);
+    }
+    state.store().archive_wallet(id).await?;
+    let updated = state.store().get_wallet(id).await?;
+    Ok(Envelope::ok(to_view(updated)))
+}
+
+/// `PATCH /v1/wallets/:id/unarchive` — unarchive a wallet (dashboard login only).
+pub async fn unarchive_wallet(
+    State(state): State<AppState>,
+    Path(id): Path<Uuid>,
+    headers: HeaderMap,
+) -> ApiResult<Json<Envelope<WalletView>>> {
+    let user_id = authenticate(&headers, &state).await?;
+    let wallet = state.store().get_wallet(id).await?;
+    if wallet.user_id != Some(user_id) {
+        return Err(ApiError::NotFound);
+    }
+    state.store().unarchive_wallet(id).await?;
+    let updated = state.store().get_wallet(id).await?;
+    Ok(Envelope::ok(to_view(updated)))
+}
+
+/// Guard check ensuring a wallet is not archived before executing a mutating operation.
+pub fn ensure_wallet_not_archived(wallet: &octo_store::Wallet) -> ApiResult<()> {
+    if wallet.is_archived() {
+        return Err(ApiError::Forbidden("wallet is archived".into()));
+    }
+    Ok(())
+}
+
 #[cfg(test)]
 mod tests {
     use super::*;
@@ -679,3 +724,4 @@ mod tests {
         }
     }
 }
+}

crates/api/src/routes/webhooks.rs

@@ -26,6 +26,8 @@ pub struct WebhookView {
     /// Returned once on creation so the caller can verify signatures.
     pub secret: String,
     pub active: bool,
+    pub recent_failure_count: i64,
+    pub last_successful_delivery_at: Option<chrono::DateTime<chrono::Utc>>,
 }
 
 /// `POST /v1/wallets/:id/webhooks`
@@ -72,6 +74,8 @@ pub async fn create_webhook(
         url: ep.url,
         secret: ep.secret,
         active: ep.active,
+        recent_failure_count: 0,
+        last_successful_delivery_at: None,
     };
     let (status, json) = Envelope::created(view);
     Ok((status, json))
@@ -168,13 +172,24 @@ pub async fn list_webhooks(
     let _ = state.store().get_wallet(wallet_id).await?;
 
     let eps = state.store().active_webhook_endpoints(wallet_id).await?;
+    let health_map = state
+        .store()
+        .wallet_webhook_delivery_health(wallet_id)
+        .await
+        .map_err(|_| ApiError::Internal)?;
+
     let views: Vec<WebhookView> = eps
         .into_iter()
-        .map(|ep| WebhookView {
-            id: ep.id,
-            url: ep.url,
-            secret: ep.secret,
-            active: ep.active,
+        .map(|ep| {
+            let health = health_map.get(&ep.id).cloned().unwrap_or_default();
+            WebhookView {
+                id: ep.id,
+                url: ep.url,
+                secret: ep.secret,
+                active: ep.active,
+                recent_failure_count: health.recent_failure_count,
+                last_successful_delivery_at: health.last_successful_delivery_at,
+            }
         })
         .collect();