fix(daemon): isolate RPC resets and add persistent diagnostics

README.md

@@ -254,6 +254,39 @@ rdc completion zsh  > ~/.zfunc/_rdc
 rdc completion fish > ~/.config/fish/completions/rdc.fish
 ```
 
+## Diagnosing RPC connection resets
+
+For an intermittent RPC error, set `RDC_DAEMON_LOG` before starting a new daemon.
+The file is appended to and retained after startup or failure. It records request
+methods, IDs, handler/send timings, connection errors, the daemon PID, and the
+loaded RenderDoc version/module path. It also enables Python's `faulthandler` to
+write native-fault tracebacks to that file where supported. For locally launched
+daemons, native stderr is retained separately at `<RDC_DAEMON_LOG>.stderr` to
+avoid competing file writers on Windows.
+
+For example, in Windows PowerShell:
+
+```powershell
+$env:RDC_DAEMON_LOG = Join-Path $PWD "rdc-rpc.log"
+rdc --session rpc-debug open .\capture.rdc --timeout 120
+rdc --session rpc-debug script .\pick_once.py *> .\rdc-client.txt
+rdc --session rpc-debug status *>> .\rdc-client.txt
+rdc --session rpc-debug close
+Remove-Item Env:RDC_DAEMON_LOG
+```
+
+Use a separate log file for each daemon, with an existing parent directory. For
+Split sessions, configure the variable on the server before launching its daemon;
+setting it on a client connected to an existing daemon has no effect. Request
+parameters and session tokens are not included in the request-stage logs, but
+native stderr and tracebacks may contain application details.
+
+Run the reproducing script once, then check `status` separately: scripts can have
+side effects and are not automatically retried. Include the client output, daemon
+log and its `.stderr` companion, exact commands, and rdc-cli version/commit in the
+bug report. Diagnostics do not guarantee a traceback for every native termination
+and do not establish the cause of a `PickPixel` failure by themselves.
+
 ## Development
 
 ```bash

src/rdc/daemon_server.py

@@ -2,8 +2,10 @@
 
 import argparse
 import atexit
+import faulthandler
 import json
 import logging
+import os
 import secrets
 import shutil
 import socket
@@ -170,6 +172,20 @@ def _cleanup_temp(state: DaemonState) -> None:
 _log = logging.getLogger("rdc.daemon")
 
 
+def _configure_diagnostics() -> logging.FileHandler | None:
+    """Enable persistent request and native-fault logs when explicitly requested."""
+    path = os.environ.get("RDC_DAEMON_LOG")
+    if not path:
+        return None
+    handler = logging.FileHandler(Path(path).expanduser(), encoding="utf-8")
+    handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s pid=%(process)d %(message)s"))
+    _log.addHandler(handler)
+    _log.setLevel(logging.INFO)
+    faulthandler.enable(file=handler.stream, all_threads=True)
+    _log.info("daemon diagnostics enabled: python=%s platform=%s", sys.version, sys.platform)
+    return handler
+
+
 class AdapterMatch(NamedTuple):
     """Adapter identity parsed from a D3D12 structured-data chunk."""
 
@@ -755,7 +771,8 @@ def run_server(  # pragma: no cover
                 conn.settimeout(10.0)
                 try:
                     line = _recv_line(conn)
-                except (ValueError, TimeoutError):
+                except (ValueError, OSError) as exc:
+                    _log.warning("RPC receive failed from %s: %s", _addr, exc)
                     continue
                 if not line:
                     continue
@@ -769,10 +786,20 @@ def run_server(  # pragma: no cover
                     }
                     try:
                         conn.sendall((json.dumps(error_resp) + "\n").encode("utf-8"))
-                    except OSError:
-                        pass
+                    except OSError as exc:
+                        _log.warning("RPC send failed: stage=parse error: %s", exc)
                     continue
+                method = request.get("method", "")
+                request_id = request.get("id")
+                started = time.perf_counter()
+                _log.info("RPC request started: method=%s id=%s", method, request_id)
                 response, running = _process_request(request, state)
+                _log.info(
+                    "RPC handler completed: method=%s id=%s elapsed_ms=%.1f",
+                    method,
+                    request_id,
+                    (time.perf_counter() - started) * 1000,
+                )
                 binary_path = response.get("result", {}).pop("_binary_path", None)
                 try:
                     payload = json.dumps(response) + "\n"
@@ -785,17 +812,29 @@ def run_server(  # pragma: no cover
                     }
                     payload = json.dumps(err_resp) + "\n"
                     binary_path = None
+                stage = "response"
                 try:
                     conn.sendall(payload.encode("utf-8"))
                     if binary_path:
-                        try:
-                            with Path(binary_path).open("rb") as bf:
-                                while chunk := bf.read(65536):
-                                    conn.sendall(chunk)
-                        except OSError:
-                            _log.warning("failed to send binary payload: %s", binary_path)
-                except OSError:
-                    pass
+                        stage = "binary payload"
+                        with Path(binary_path).open("rb") as bf:
+                            while chunk := bf.read(65536):
+                                conn.sendall(chunk)
+                except OSError as exc:
+                    _log.warning(
+                        "RPC send failed: method=%s id=%s stage=%s: %s",
+                        method,
+                        request_id,
+                        stage,
+                        exc,
+                    )
+                else:
+                    _log.info(
+                        "RPC response sent: method=%s id=%s elapsed_ms=%.1f",
+                        method,
+                        request_id,
+                        (time.perf_counter() - started) * 1000,
+                    )
                 last_activity = time.time()
 
     # The shutdown RPC runs cleanup via _handle_shutdown before exiting the
@@ -818,6 +857,7 @@ def main() -> None:  # pragma: no cover
     parser.add_argument("--remote-url", default=None)
     parser.add_argument("--gpu", default=None)
     args = parser.parse_args()
+    _configure_diagnostics()
 
     state = DaemonState(
         capture=args.capture, current_eid=0, token=args.token, gpu_pref=args.gpu or ""
@@ -832,6 +872,14 @@ def main() -> None:  # pragma: no cover
             sys.stderr.write(f"error: {err}\n")
             sys.exit(1)
 
+    if _log.isEnabledFor(logging.INFO):
+        _log.info(
+            "replay ready: renderdoc=%s module=%s api=%s remote=%s",
+            _detect_version(state.rd) if state.rd is not None else None,
+            getattr(state.rd, "__file__", None),
+            state.api_name,
+            state.is_remote,
+        )
     run_server(
         host=args.host,
         port=args.port,

src/rdc/services/session_service.py

@@ -74,8 +74,16 @@ def start_daemon(
     # replay the daemon's verbose upload logging would fill the OS pipe buffer
     # and block before serving its first ping if no one drains it, which deadlocks
     # wait_for_ping. A file has no buffer limit and we read its tail on failure.
-    stderr_file = tempfile.NamedTemporaryFile(  # noqa: SIM115
-        mode="w+", prefix="rdc-daemon-", suffix=".stderr", delete=False
+    diagnostic_path = os.environ.get("RDC_DAEMON_LOG")
+    # Keep native stderr separate from the diagnostic FileHandler. On Windows,
+    # subprocesses inherit the handle's position but not the CRT append flag;
+    # a second writer to the same file could overwrite newer log records.
+    stderr_file = (
+        Path(f"{diagnostic_path}.stderr").expanduser().open("a", encoding="utf-8")
+        if diagnostic_path
+        else tempfile.NamedTemporaryFile(  # noqa: SIM115
+            mode="w+", prefix="rdc-daemon-", suffix=".stderr", delete=False
+        )
     )
     proc = subprocess.Popen(
         cmd,
@@ -85,12 +93,13 @@ def start_daemon(
         **_platform.popen_flags(),
     )
     proc._rdc_stderr_path = stderr_file.name  # type: ignore[attr-defined]
+    proc._rdc_preserve_stderr = bool(diagnostic_path)  # type: ignore[attr-defined]
     stderr_file.close()
     return proc
 
 
 def _read_daemon_stderr(proc: subprocess.Popen[str]) -> str:
-    """Return the tail of the daemon's redirected stderr, then remove the file."""
+    """Read the stderr tail, removing only automatically-created temporary logs."""
     path = getattr(proc, "_rdc_stderr_path", None)
     if not path:
         return ""
@@ -99,22 +108,22 @@ def _read_daemon_stderr(proc: subprocess.Popen[str]) -> str:
     except OSError:
         return ""
     finally:
-        try:
-            Path(path).unlink()
-        except OSError:
-            pass
+        if getattr(proc, "_rdc_preserve_stderr", False) is not True:
+            try:
+                Path(path).unlink()
+            except OSError:
+                pass
     return "\n".join(text.splitlines()[-20:]).strip()
 
 
 def _discard_daemon_stderr(proc: subprocess.Popen[str]) -> None:
     """Best-effort removal of the daemon's stderr temp file after a successful start.
 
-    On POSIX the daemon keeps writing to the now-unlinked inode; on Windows the
-    file is still open so unlink may fail and is silently ignored (the OS reclaims
-    it on process exit).
+    Explicit diagnostic logs are retained. Otherwise, on POSIX the daemon keeps
+    writing to the now-unlinked inode; on Windows an open file may prevent unlink.
     """
     path = getattr(proc, "_rdc_stderr_path", None)
-    if path:
+    if path and getattr(proc, "_rdc_preserve_stderr", False) is not True:
         try:
             Path(path).unlink()
         except OSError: