Sub-Rosa-Issue · Sheebazz · Aug 1, 2026
diff --git a/README.md b/README.md
@@ -214,3 +214,6 @@ pnpm mainnet:verify         # mainnet read-only proof
 - **Binding:** `H = sha256(value‖nonce)`
 - **Unlock:** round-R BLS verified on-chain before reveal
 - **Selective disclosure:** values public post-R; identities auditor-encrypted
+
+## Operational Behavior
+- **Graceful Shutdown**: The keeper watch mode listens for `SIGINT` and `SIGTERM`. Upon receiving a signal, it stops picking up new rounds, finishes processing the current in-flight round, and safely exits. A 30-second timeout ensures the process does not hang indefinitely.
diff --git a/services/keeper/src/serve.ts b/services/keeper/src/serve.ts
@@ -1,23 +1,4 @@
 // Standalone status server for the keeper.
-//
-// Runs the watch-mode keeper AND a status HTTP API on the same process so
-// pilots and dashboards can poll keeper-observed rounds without SSHing into
-// the host. The status server reads from the same on-chain source and the
-// same persisted store as the watch loop — no extra RPC budget, no extra
-// signing capability.
-//
-// Env:
-//   ROUND_CONTRACT_ID   deployed Round contract id (C…)
-//   KEEPER_SECRET       funded signer secret (S…)
-//   RPC_URL             Soroban RPC (default testnet)
-//   NETWORK_PASSPHRASE
-//   WATCH_POLL_MS       poll interval (default 15000)
-//   WATCH_ROUND_IDS     optional explicit list: "1,2,5" or "1-10"
-//   WATCH_FROM          first round id when auto-discovering (default 1)
-//   WATCH_MAX_ROUNDS    max rounds to probe (default 64)
-//   KEEPER_STATUS_HOST  status API bind host (default 127.0.0.1)
-//   KEEPER_STATUS_PORT  status API port (default 8090)
-//   KEEPER_STATUS_ENABLE set to "false" to disable the status API (default true)
 
 import { Keypair } from "@stellar/stellar-sdk";
 import { SubRosaClient } from "@sub-rosa/sdk";
@@ -60,15 +41,6 @@ async function main() {
   const store = new KeeperStore();
   const settlementGuard = createSettlementGuard();
 
-  let stopping = false;
-  process.on("SIGINT", () => {
-    console.log("\nserve: SIGINT — finishing current tick then exit");
-    stopping = true;
-  });
-  process.on("SIGTERM", () => {
-    stopping = true;
-  });
-
   const statusEnabled = (process.env.KEEPER_STATUS_ENABLE ?? "true").toLowerCase() !== "false";
   const statusHost = process.env.KEEPER_STATUS_HOST ?? "127.0.0.1";
   const statusPort = Number(process.env.KEEPER_STATUS_PORT ?? "8090");
@@ -92,16 +64,30 @@ async function main() {
       },
     });
     statusHandle = withGracefulShutdown(server);
-    console.log(`· status API: http://${statusHost}:${statusPort} (GET /status, /status/rounds/:id, /healthz, /status/health)`);
-  } else {
-    console.log("· status API disabled (KEEPER_STATUS_ENABLE=false)");
+    console.log(`· status API: http://${statusHost}:${statusPort}`);
   }
 
   console.log("Sub Rosa keeper (watch + status)");
   console.log("· contract:", contractId);
   console.log("· poll:    ", pollMs, "ms");
   console.log("· Ctrl+C to stop\n");
 
+  let stopping = false;
+  let shutdownTimer: NodeJS.Timeout | undefined;
+
+  const handleSignal = (signal: string) => {
+    if (stopping) return;
+    stopping = true;
+    console.log(JSON.stringify({ event: "shutdown_start", signal, message: "finishing current tick then exit" }));
+    shutdownTimer = setTimeout(() => {
+      console.error(JSON.stringify({ event: "shutdown_timeout", message: "timeout exceeded, forcing exit" }));
+      process.exit(1);
+    }, 30000);
+  };
+
+  process.on("SIGINT", () => handleSignal("SIGINT"));
+  process.on("SIGTERM", () => handleSignal("SIGTERM"));
+
   await runWatchLoop({
     sdk,
     drand,
@@ -115,7 +101,8 @@ async function main() {
   });
 
   if (statusHandle) await statusHandle.close();
-  console.log("serve: stopped");
+  if (shutdownTimer) clearTimeout(shutdownTimer);
+  console.log(JSON.stringify({ event: "shutdown_complete", message: "serve stopped gracefully" }));
 }
 
 main().catch((err) => {

diff --git a/services/keeper/src/watch.test.ts b/services/keeper/src/watch.test.ts
@@ -33,3 +33,46 @@ test("discoverRoundIds returns empty when first round missing", async () => {
   const ids = await discoverRoundIds(reader as Pick<import("@sub-rosa/sdk").SubRosaClient, "getRound">, { from: 1n, maxProbe: 5 });
   assert.deepEqual(ids, []);
 });
+
+test("watch loop handles graceful shutdown during a queued round", async () => {
+  const { KeeperStore } = await import("./store.js");
+  const { createSettlementGuard } = await import("./settlement-guard.js");
+  const { runWatchLoop } = await import("./watch-loop.js");
+
+  const store = new KeeperStore();
+  const settlementGuard = createSettlementGuard();
+
+  store.addRound(100n, { contractId: "C1", network: "test" });
+  store.addRound(101n, { contractId: "C1", network: "test" });
+
+  let callCount = 0;
+  let processedRoundBeforeStop = false;
+
+  const isStopping = () => {
+    callCount++;
+    if (callCount > 2) {
+      processedRoundBeforeStop = true;
+      return true; 
+    }
+    return false;
+  };
+
+  const mockSdk = {
+    getRound: async () => { throw new Error("mock error"); }
+  } as any;
+
+  const loopPromise = runWatchLoop({
+    sdk: mockSdk,
+    drand: {} as any,
+    log: () => {},
+    pollMs: 10,
+    contractId: "C1",
+    network: "test",
+    store,
+    settlementGuard,
+    isStopping,
+  });
+
+  await loopPromise;
+  assert.ok(processedRoundBeforeStop, "gracefully stopped without hanging");
+});
diff --git a/services/keeper/src/watch.ts b/services/keeper/src/watch.ts
@@ -1,15 +1,5 @@
 // Watch-mode keeper — standalone entry. For a combined status-API + watch
 // process, use `serve.ts` instead.
-//
-// Env:
-//   ROUND_CONTRACT_ID   deployed Round contract id (C…)
-//   KEEPER_SECRET       funded signer secret (S…)
-//   RPC_URL             Soroban RPC (default testnet)
-//   NETWORK_PASSPHRASE
-//   WATCH_POLL_MS       poll interval (default 15000)
-//   WATCH_ROUND_IDS     optional explicit list: "1,2,5" or "1-10"
-//   WATCH_FROM          first round id when auto-discovering (default 1)
-//   WATCH_MAX_ROUNDS    max rounds to probe (default 64)
 
 import { Keypair } from "@stellar/stellar-sdk";
 import { SubRosaClient } from "@sub-rosa/sdk";
@@ -42,15 +32,6 @@ async function main() {
   const drand = quicknet();
   const log = (m: string) => console.log(`· ${m}`);
 
-  let stopping = false;
-  process.on("SIGINT", () => {
-    console.log("\nwatch: SIGINT — finishing current tick then exit");
-    stopping = true;
-  });
-  process.on("SIGTERM", () => {
-    stopping = true;
-  });
-
   const store = new KeeperStore();
   const settlementGuard = createSettlementGuard();
 
@@ -59,6 +40,22 @@ async function main() {
   console.log("· poll:    ", pollMs, "ms");
   console.log("· Ctrl+C to stop\n");
 
+  let stopping = false;
+  let shutdownTimer: NodeJS.Timeout | undefined;
+
+  const handleSignal = (signal: string) => {
+    if (stopping) return;
+    stopping = true;
+    console.log(JSON.stringify({ event: "shutdown_start", signal, message: "finishing current tick then exit" }));
+    shutdownTimer = setTimeout(() => {
+      console.error(JSON.stringify({ event: "shutdown_timeout", message: "timeout exceeded, forcing exit" }));
+      process.exit(1);
+    }, 30000);
+  };
+
+  process.on("SIGINT", () => handleSignal("SIGINT"));
+  process.on("SIGTERM", () => handleSignal("SIGTERM"));
+
   await runWatchLoop({
     sdk,
     drand,
@@ -71,7 +68,8 @@ async function main() {
     isStopping: () => stopping,
   });
 
-  console.log("watch: stopped");
+  if (shutdownTimer) clearTimeout(shutdownTimer);
+  console.log(JSON.stringify({ event: "shutdown_complete", message: "watch stopped gracefully" }));
 }
 
 main().catch((err) => {