From 9040cbae7d53e0c75a4032527fa6342760c8e168 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 04:50:24 -0500 Subject: [PATCH 1/3] fix(forge-import): retry a GitHub listing whose HTTP/2 stream was reset A dashpay/dash collab import failed 9 minutes in on the 213-page review-comment listing: gh reported 'stream error: stream ID 119; CANCEL; received from peer', which was not among the transient markers, so the run ended at once. It is now retried like a 502. The markers move into is_transient, with a unit test. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/forge-import/src/github.rs | 43 +++++++++++++++++++++---------- 1 file changed, 30 insertions(+), 13 deletions(-) diff --git a/crates/forge-import/src/github.rs b/crates/forge-import/src/github.rs index 445aa570c..6aa1bf52e 100644 --- a/crates/forge-import/src/github.rs +++ b/crates/forge-import/src/github.rs @@ -741,19 +741,7 @@ fn gh_output_with_retry(path: &str, rest: &[&str], what: &str) -> Result Result bool { + [ + "connection reset", + "timeout", + "TLS handshake", + "temporary failure", + "EOF", + "stream error", + "502", + "503", + "504", + ] + .iter() + .any(|m| stderr.contains(m)) +} + /// How long to wait out a rate limit on `path` (`stderr` is the refused call's): until the /// limit resets (capped at 15 minutes), else a growing default (a secondary limit asks for a /// minute or so). @@ -985,6 +992,16 @@ mod tests { assert_eq!(rate_limit_headers("HTTP/2.0 403 Forbidden\n\n{}"), None); assert_eq!(rate_limit_headers(""), None); } + /// An HTTP/2 stream GitHub reset mid-listing is retried, as a 502 is; a 404 is not. + #[test] + fn a_reset_stream_is_transient() { + assert!(is_transient( + "stream error: stream ID 119; CANCEL; received from peer" + )); + assert!(is_transient("gh: Server Error (HTTP 502)")); + assert!(!is_transient("gh: Not Found (HTTP 404)")); + } + /// F-7 against a whole recorded listing: set `FORGE_IMPORT_GH_NDJSON` to a file of /// `gh api repos/octocat/Hello-World/pulls/comments --paginate --jq '.[]'` output. #[test] From 37ea3741fbceaf301cecb1d88a036cd1742204b7 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 05:05:50 -0500 Subject: [PATCH 2/3] fix(forge-import): read a 100-per-page GitHub listing a page at a time gh --paginate restarts a listing from page 1 whenever any page fails. dashpay/dash's 213-page review-comment listing got a 502 or a reset stream somewhere past page 100 on every attempt, so both collab runs died after 4 attempts (about 13 minutes each). A per_page=100 listing is now read page by page, each page with its own retries, stopping at the first short page; other listings keep --paginate. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/forge-import/src/github.rs | 47 +++++++++++++++++++++++-------- 1 file changed, 35 insertions(+), 12 deletions(-) diff --git a/crates/forge-import/src/github.rs b/crates/forge-import/src/github.rs index 6aa1bf52e..a4cab2327 100644 --- a/crates/forge-import/src/github.rs +++ b/crates/forge-import/src/github.rs @@ -825,19 +825,42 @@ fn api_json(path: &str) -> Result> { Ok(gh_output_with_retry(path, &[], &format!("`gh api {path}`"))?.stdout) } +/// A listing's elements, one JSON document per line. A `per_page=100` listing is read a page +/// at a time, each page retried on its own: `gh --paginate` restarts from page 1 whenever any +/// page fails, and dashpay/dash's 213-page review-comment listing then never finished (GitHub +/// answered a 502 or reset the stream somewhere past page 100 on every attempt). fn api_list_lines(path: &str) -> Result> { - let out = gh_output_with_retry( - path, - &["--paginate", "--jq", ".[]"], - &format!("`gh api {path}`"), - )?; - let text = String::from_utf8(out.stdout).context("gh api output was not UTF-8")?; - Ok(text - .lines() - .map(str::trim) - .filter(|l| !l.is_empty()) - .map(str::to_string) - .collect()) + let lines = |out: std::process::Output| -> Result> { + let text = String::from_utf8(out.stdout).context("gh api output was not UTF-8")?; + Ok(text + .lines() + .map(str::trim) + .filter(|l| !l.is_empty()) + .map(str::to_string) + .collect()) + }; + if !path.contains("per_page=100") { + return lines(gh_output_with_retry( + path, + &["--paginate", "--jq", ".[]"], + &format!("`gh api {path}`"), + )?); + } + let mut all = Vec::new(); + for page in 1.. { + let paged = format!("{path}&page={page}"); + let batch = lines(gh_output_with_retry( + &paged, + &["--jq", ".[]"], + &format!("`gh api {paged}`"), + )?)?; + let last = batch.len() < 100; + all.extend(batch); + if last { + break; + } + } + Ok(all) } /// ISO 8601 UTC (`YYYY-MM-DDTHH:MM:SSZ`) to unix seconds; `0` when unparseable (provenance From ff7560ae359706ec8b914f5892319e2d6b372d21 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 05:21:00 -0500 Subject: [PATCH 3/3] fix(forge-import): retry a GitHub page cut short, and give each call 6 attempts The page-at-a-time read got to page 89 of dashpay/dash's review-comment listing, then gh failed with 'unexpected end of JSON input' (a truncated response; the same page read cleanly three times right after). That is now transient too, and a call gets 6 attempts instead of 4 (backoff 5 to 25 s): a retry costs one page now, not the whole listing. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/forge-import/src/github.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/crates/forge-import/src/github.rs b/crates/forge-import/src/github.rs index a4cab2327..0870a3d66 100644 --- a/crates/forge-import/src/github.rs +++ b/crates/forge-import/src/github.rs @@ -696,7 +696,7 @@ mod base64_lite { /// Run `gh api `, retrying transient network failures; 4xx and auth failures /// fail at once. fn gh_output_with_retry(path: &str, rest: &[&str], what: &str) -> Result { - const ATTEMPTS: u32 = 4; + const ATTEMPTS: u32 = 6; // Waits for a rate limit, each up to 15 minutes: enough to reach the hourly reset. Every // call has its own, so a run of many small reads larger than one hour's quota (dashpay/dash's // ~7,000 per-PR review reads) goes on after the reset instead of failing. @@ -763,6 +763,7 @@ fn is_transient(stderr: &str) -> bool { "temporary failure", "EOF", "stream error", + "unexpected end of JSON input", "502", "503", "504", @@ -1022,6 +1023,8 @@ mod tests { "stream error: stream ID 119; CANCEL; received from peer" )); assert!(is_transient("gh: Server Error (HTTP 502)")); + // A page cut short: gh's --jq meets truncated JSON (page 89 of the same listing). + assert!(is_transient("unexpected end of JSON input")); assert!(!is_transient("gh: Not Found (HTTP 404)")); }