fix(agent): keep a slow Simlock daemon instead of restarting it

packages/agent/src/index.ts

@@ -490,12 +490,18 @@ async function restartDaemon(): Promise<string | null> {
   warn(`Simlock daemon is not answering; restarting it (${used + 1}/${DAEMON_RESTART_LIMIT})`);
   await state.simlock!.logs();
   await state.simlock!.stop().catch(() => {});
-  try {
-    await state.simlock!.start();
-  } catch (err) {
-    return `Simlock daemon restart failed: ${message(err).slice(0, 150)}`;
+  // A start right after a slow stop can still find the old daemon going away
+  // ("Daemon connection closed"), so it gets one more try.
+  for (let attempt = 1; ; attempt++) {
+    try {
+      await state.simlock!.start();
+      return null;
+    } catch (err) {
+      if (attempt >= 2) return `Simlock daemon restart failed: ${message(err).slice(0, 150)}`;
+      warn(`Simlock daemon start failed (${message(err).slice(0, 150)}); retrying`);
+      await interruptibleSleep(15_000);
+    }
   }
-  return null;
 }
 
 async function checkHardExit(): Promise<void> {
@@ -529,13 +535,19 @@ async function supervise(): Promise<never> {
     const status = await state.simlock!.status();
     if (!status) {
       daemonProbeFailures += 1;
-      warn(`simlock status failed (${daemonProbeFailures}/${DAEMON_PROBE_FAILURE_LIMIT})`);
+      warn(
+        `simlock status failed (${daemonProbeFailures}/${DAEMON_PROBE_FAILURE_LIMIT}): ${state.simlock!.statusProblem()}`,
+      );
       // Unknown load is not evidence of idleness.
       lastWorkAt = Date.now();
       if (daemonProbeFailures >= DAEMON_PROBE_FAILURE_LIMIT) {
         daemonProbeFailures = 0;
-        const failure = await restartDaemon();
-        if (failure) await shutdown(failure, 1);
+        if (await state.simlock!.alive()) {
+          warn("Simlock is slow but answers its health check; not restarting it");
+        } else {
+          const failure = await restartDaemon();
+          if (failure) await shutdown(failure, 1);
+        }
       }
     } else {
       daemonProbeFailures = 0;

packages/agent/src/simlock.ts

@@ -482,6 +482,23 @@ export function createSimlock(bins: SimlockBins) {
     status,
     logs,
 
+    /** Why the last status read failed; empty after a good read. */
+    statusProblem: () => statusProblem,
+
+    /**
+     * Whether the daemon answers its health route within `timeoutMs`. A host
+     * busy wiping and re-warming a device can make `simlock status` outlast
+     * its limit while the daemon is fine; restarting it then lost the run.
+     */
+    async alive(timeoutMs = 60_000): Promise<boolean> {
+      try {
+        const res = await fetch(SIMLOCK_HEALTH_URL, { signal: AbortSignal.timeout(timeoutMs) });
+        return res.ok;
+      } catch {
+        return false;
+      }
+    },
+
     async configure(options: Parameters<typeof simlockSettings>[0]): Promise<void> {
       for (const [key, value] of simlockSettings(options)) {
         const res = await simlock(["config", "set", key, value]);
@@ -526,7 +543,7 @@ export function createSimlock(bins: SimlockBins) {
     },
 
     async stop(): Promise<void> {
-      await simlock(["daemon", "stop"], 20_000);
+      await simlock(["daemon", "stop"], 90_000);
     },
 
     /**
@@ -590,6 +607,9 @@ export function simlockLoad(status: SimlockStatus): { leases: number; queued: nu
   return { leases: status.leases?.length ?? 0, queued: status.queueDepth ?? 0 };
 }
 
+/** Simlock's anonymous liveness route on the run's daemon. */
+export const SIMLOCK_HEALTH_URL = "http://127.0.0.1:4701/v1/healthz";
+
 /** Warm devices ready to lease right now. */
 export function warmReady(status: SimlockStatus): number {
   return (status.warmPool?.targets ?? []).reduce((sum, target) => sum + target.ready, 0);