From 7b99098811cf619c18efec0eab0bddea90975320 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 08:19:39 -0500 Subject: [PATCH 1/3] feat(import): pipeline each item's writes behind its ordered create Issue and PR creates stay one strict sequence in upstream order (dense numbering), each confirmed before the next. Each item's comments, reviews, label events and transitions run as one sequential job, in up to --concurrency lanes (default 8; 1 is the old one-write-at-a-time run). The budget reserves each admitted write's estimate on top of the measured drop until a balance read covers it, the first run-level error stops new writes and drains the ones in flight, and a progress line reports placed/complete items, documents and spend. A Chain trait lets the sink run against an in-memory recorded chain: pipelined vs sequential, crash-and-resume, spend cap and refused-item tests. Co-Authored-By: Claude Opus 5.5 (1M context) --- Cargo.lock | 1 + crates/dg/src/import.rs | 10 + crates/forge-import/Cargo.toml | 3 + crates/forge-import/src/budget.rs | 117 +++- crates/forge-import/src/chain.rs | 375 +++++++++++ crates/forge-import/src/dest.rs | 26 +- crates/forge-import/src/importer.rs | 5 + crates/forge-import/src/lib.rs | 8 + crates/forge-import/src/main.rs | 23 + crates/forge-import/src/pipeline.rs | 477 +++++++++++++ crates/forge-import/src/sink.rs | 923 ++++++++++++++++---------- crates/forge-import/src/sink_tests.rs | 805 ++++++++++++++++++++++ 12 files changed, 2408 insertions(+), 365 deletions(-) create mode 100644 crates/forge-import/src/chain.rs create mode 100644 crates/forge-import/src/pipeline.rs create mode 100644 crates/forge-import/src/sink_tests.rs diff --git a/Cargo.lock b/Cargo.lock index 1c2ee43cd..689f9fd10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1900,6 +1900,7 @@ dependencies = [ "anyhow", "clap", "forge-core", + "futures", "hex", "reqwest", "serde", diff --git a/crates/dg/src/import.rs b/crates/dg/src/import.rs index d42d01cf2..0b622b68b 100644 --- a/crates/dg/src/import.rs +++ b/crates/dg/src/import.rs @@ -47,6 +47,15 @@ pub struct ImportArgs { /// Import at most this many issues and PRs (0 = all). #[arg(long, default_value_t = 0)] pub limit: usize, + /// Parallel lanes for each issue's and PR's comments, reviews, labels and state (issues + /// and PRs are always created one at a time, in upstream order). 1: one write at a time. + #[arg( + long, + value_name = "N", + default_value_t = forge_import::pipeline::DEFAULT_LANES, + value_parser = forge_import::pipeline::parse_lanes + )] + pub concurrency: usize, } fn max_spend(dash: Option) -> Result> { @@ -128,6 +137,7 @@ pub async fn import(ctx: &Ctx, a: &ImportArgs) -> Result<()> { limit: a.limit, network: ctx.target.clone(), key: ctx.identity_path.clone(), + concurrency: a.concurrency, }; report(ctx, &importer::run(&cfg).await) } diff --git a/crates/forge-import/Cargo.toml b/crates/forge-import/Cargo.toml index 094465592..00d7380c6 100644 --- a/crates/forge-import/Cargo.toml +++ b/crates/forge-import/Cargo.toml @@ -24,10 +24,13 @@ serde.workspace = true serde_json.workspace = true hex = "0.4" sha2 = "0.10" +futures = "0.3" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } [dev-dependencies] tempfile = "3" +# Paused time for the pipeline tests (`#[tokio::test(start_paused = true)]`). +tokio = { workspace = true, features = ["time", "test-util"] } [lints] workspace = true diff --git a/crates/forge-import/src/budget.rs b/crates/forge-import/src/budget.rs index f7d0b29e5..a8da6db1c 100644 --- a/crates/forge-import/src/budget.rs +++ b/crates/forge-import/src/budget.rs @@ -6,6 +6,16 @@ //! the estimates of the writes made and the identity's measured balance drop, so an //! estimate that runs low cannot carry the run past the cap for long: every //! [`Budget::reconcile`] pulls the measured figure in. +//! +//! **Writes in flight together** (the pipelined write phase, [`crate::pipeline`]) reserve +//! their estimate when admitted ([`Budget::reserve`]) and hold it until a balance read that +//! began after they finished has been folded in ([`Budget::finish`], [`Budget::mark`], +//! [`Budget::measured`]). Reads may come back in any order: each covers the writes finished +//! before it began, and the furthest such point folded in is what counts. +//! Admission is one synchronous step under the ledger's lock, so two writes can never both +//! pass on the same headroom, and "spent so far" never forgets a write the measured drop may +//! not show yet: it is the larger of the estimates and the measured drop plus every reserved +//! write's estimate. With one write at a time this is exactly the check above. use anyhow::{bail, Result}; @@ -126,6 +136,13 @@ pub struct Budget { estimated: u64, measured: u64, balance_start: Option, + /// Reserved writes still running ([`Self::reserve`]), credits. + in_flight: u64, + /// Every reserved write that finished, a running total ([`Self::finish`]), credits. + finished: u64, + /// How much of `finished` the balance reads folded in so far cover: the furthest + /// [`Self::mark`] handed back to [`Self::measured`]. + covered: u64, } impl Budget { @@ -136,6 +153,9 @@ impl Budget { estimated: 0, measured: 0, balance_start: None, + in_flight: 0, + finished: 0, + covered: 0, } } @@ -144,9 +164,13 @@ impl Budget { self.balance_start = Some(balance); } - /// What this run has spent, as far as it can tell (credits). + /// What this run has spent, as far as it can tell (credits): the estimates of every write + /// admitted, or the measured drop plus the reserved writes it may not show yet, whichever + /// is larger. pub fn spent(&self) -> u64 { - self.estimated.max(self.measured) + let unmeasured = self.finished.saturating_sub(self.covered); + let pending = self.in_flight.saturating_add(unmeasured); + self.estimated.max(self.measured.saturating_add(pending)) } /// Whether a write of `credits` would fit under the cap now. @@ -217,6 +241,37 @@ impl Budget { Ok(()) } + /// [`Self::charge`] for a write that may run beside others: its estimate stays reserved + /// on top of the measured drop until [`Self::finish`] and a balance read begun after it + /// ([`Self::mark`], [`Self::measured`]). + pub fn reserve(&mut self, credits: u64, what: impl Into) -> Result<()> { + self.charge(credits, what)?; + self.in_flight = self.in_flight.saturating_add(credits); + Ok(()) + } + + /// A reserved write of `credits` finished (landed or not): its estimate waits for the + /// next balance read. + pub fn finish(&mut self, credits: u64) { + let credits = credits.min(self.in_flight); + self.in_flight -= credits; + self.finished = self.finished.saturating_add(credits); + } + + /// Where a balance read starting now stands: it covers every reserved write finished so + /// far. Hand it to [`Self::measured`] with the read's result. + pub fn mark(&self) -> u64 { + self.finished + } + + /// Fold in a balance read that began at `mark`: the writes finished before it are in the + /// measured drop now, and no longer held on top of it. An older read folded in after a + /// newer one releases nothing more. + pub fn measured(&mut self, balance_now: u64, mark: u64) { + self.reconcile(balance_now); + self.covered = self.covered.max(mark); + } + /// Take back an admitted charge whose write stored nothing (a taken number). The /// measured drop still counts whatever fee was really paid. pub fn refund(&mut self, credits: u64) { @@ -271,6 +326,64 @@ mod tests { assert_eq!(b.spent(), 95); } + /// Writes in flight together: each is admitted against the others' reservations, and a + /// reservation stays on top of the measured drop until a balance read that began after the + /// write finished has been folded in. + #[test] + fn writes_in_flight_together_hold_their_reservations() { + let mut b = Budget::new(Some(100)); + b.start(1_000); + b.reserve(40, "a").unwrap(); + b.reserve(40, "b").unwrap(); + // A third would cross the cap on the reservations alone. + assert!(b.reserve(40, "c").is_err()); + assert_eq!(b.spent(), 80); + // `a` finishes; a balance read starts (covering `a`), and meanwhile `b` finishes. + b.finish(40); + let mark = b.mark(); + b.finish(40); + // The chain says 70 was spent: `a` really cost 70, and `b` may not show yet. + b.measured(930, mark); + assert_eq!(b.spent(), 110, "measured 70 plus b's reservation 40"); + assert!( + b.reserve(1, "d").is_err(), + "over the cap: nothing more is admitted" + ); + // The next read covers `b` too: only the measured drop is left. + let mark = b.mark(); + b.measured(920, mark); + assert_eq!(b.spent(), 80); + assert!(b.reserve(20, "e").is_ok()); + // One write at a time is the plain check. + let mut one = Budget::new(Some(100)); + one.start(1_000); + one.reserve(10, "x").unwrap(); + one.finish(10); + let mark = one.mark(); + one.measured(905, mark); + assert_eq!(one.spent(), 95); + } + + /// Two balance reads overlap and come back out of order: the older one must not release + /// a write that only the newer one's start came after, nor one that neither covers. + #[test] + fn balance_reads_folded_in_out_of_order_release_only_what_they_cover() { + let mut b = Budget::new(Some(1_000)); + b.start(10_000); + for w in [10, 15, 5] { + b.reserve(w, "w").unwrap(); + } + b.finish(10); + let r1 = b.mark(); // covers the 10 + b.finish(15); + let r2 = b.mark(); // covers the 10 and the 15 + b.finish(5); // neither read covers it + b.measured(9_975, r2); + b.measured(9_990, r1); + // Measured 25 (the larger drop), plus the 5 no read covers. + assert_eq!(b.spent(), 30); + } + #[test] fn the_plan_is_refused_whole_before_any_write() { let b = Budget::new(Some(50)); diff --git a/crates/forge-import/src/chain.rs b/crates/forge-import/src/chain.rs new file mode 100644 index 000000000..e04203bca --- /dev/null +++ b/crates/forge-import/src/chain.rs @@ -0,0 +1,375 @@ +//! What the [`crate::sink::Sink`] reads from and writes to the destination, as one trait: the +//! production [`CollabChain`] over forge-core's `Collab`, and an in-memory chain in the tests +//! (`crate::sink_tests`), so the importer's diff, resume and pipelining run offline. +//! +//! Every method is one of `Collab`'s (or the client's), narrowed to what the sink uses: a +//! target's state as the sink compares it ([`Current`]), and a thread's comments and reviews +//! as who wrote them and which source item each copies ([`Written`]). + +use std::collections::BTreeSet; + +use forge_core::collab::v2::{ + Collab, Created, ImportedRow, ImportedTarget, Provenance, Target, TargetKind, +}; +use forge_core::collab::{CommentAnchor, Imported, Label, Release, ReleaseInput, Verdict}; +use forge_core::history::Freshness; +use forge_core::platform::PlatformClient; +use forge_core::rules::v2::{issue_state_v2, pr_state_v2, TransitionMove, Visibility}; +use forge_core::rules::{EventKind, MergeBaseTips}; +use forge_core::scope::RepoRef; +use forge_core::Result; + +use crate::model::SrcRelease; +use crate::sealed_release::{CollabDest, ReleaseStorage, ReleaseTargets}; +use crate::sink::Ledger; + +/// A target's state on chain: its state code (the sum of its transitions) and its labels. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Current { + /// The state code. + pub code: i64, + /// The labels, as every reader folds the target's events. + pub labels: BTreeSet, +} + +/// A comment or review already on a target: who wrote it and which source item it copies. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Written { + /// Its author (a review's reviewer). + pub author: String, + /// Its `imported.url`, when it was imported. + pub url: Option, +} + +/// The destination operations the sink needs. Async methods on `&self`: the pipelined write +/// phase calls them from several lanes at once. +#[allow(async_fn_in_trait)] // used through generics only, never boxed or sent +pub trait Chain { + /// The `kind` documents of `repo` whose `upstreamNumber` is `n`. + async fn upstream_targets( + &self, + repo: &RepoRef, + kind: TargetKind, + n: u32, + ) -> Result>; + /// The `kind` documents carrying an `upstreamNumber`, highest first: one page, or `all`. + async fn upstream_numbered( + &self, + repo: &RepoRef, + kind: TargetKind, + all: bool, + ) -> Result>; + /// Every issue and PR of `repo` with its provenance. + async fn imported_targets(&self, repo: &RepoRef) -> Result>; + /// `target`'s state code and labels. + async fn current(&self, repo: &RepoRef, target: &Target) -> Result; + /// The comments on `target_id`. + async fn comments(&self, repo: &RepoRef, target_id: &str) -> Result>; + /// The reviews of `patch_id`. + async fn reviews(&self, repo: &RepoRef, patch_id: &str) -> Result>; + /// The destination's members (maintainers and writers). + async fn members(&self, repo: &RepoRef) -> Result>; + /// The label definitions. + async fn labels(&self, repo: &RepoRef) -> Result>; + /// The release of each live tag. + async fn releases(&self, repo: &RepoRef) -> Result>; + /// The tips a merge into `base_ref` (of a PR opened at `opened_at`) is checked against. + async fn merge_base( + &self, + repo: &RepoRef, + base_ref: &str, + opened_at: u64, + freshness: Freshness, + ) -> Result; + /// Forget cached ref updates (this process pushed since they were read). + fn refs_changed(&self); + /// `identity`'s balance now, when it can be read. + async fn balance(&self, identity: &str) -> Option; + + /// Create an imported issue or PR at the dense next number. + async fn create_imported( + &self, + repo: &RepoRef, + what: ImportedTarget<'_>, + from: Provenance<'_>, + ) -> Result; + /// Comment on `target_id`. + async fn comment( + &self, + repo: &RepoRef, + target_id: &str, + body: &str, + anchor: Option<&CommentAnchor>, + imported: &Imported, + ) -> Result; + /// Review `patch_id` (always a comment verdict: see [`crate::sink`]). + async fn review( + &self, + repo: &RepoRef, + patch_id: &str, + commit_oid: &[u8], + body: &str, + imported: &Imported, + ) -> Result; + /// A member label event on `target`. + async fn post_event( + &self, + repo: &RepoRef, + target: &Target, + kind: EventKind, + value: &str, + ) -> Result; + /// The state transition `mv` on `target`. + async fn write_transition( + &self, + repo: &RepoRef, + target: &Target, + mv: &TransitionMove, + merge_oid: Option<&[u8]>, + ) -> Result; + /// Define a label. + async fn create_label( + &self, + repo: &RepoRef, + name: &str, + color: &str, + description: &str, + ) -> Result; + /// Publish a release. + async fn create_release(&self, repo: &RepoRef, input: &ReleaseInput) -> Result; + /// A private destination's releases, sealed ([`crate::sealed_release::sync`]), their files + /// and asset lists stored on `storage`. + async fn sync_sealed_releases( + &self, + ledger: &mut Ledger<'_>, + repo: &RepoRef, + releases: &[SrcRelease], + storage: Option<&ReleaseStorage>, + ) -> anyhow::Result<()>; +} + +/// The destination on Dash Platform, through forge-core's `Collab`. +pub struct CollabChain<'a> { + collab: Collab<'a>, + client: &'a PlatformClient, +} + +impl<'a> CollabChain<'a> { + /// The destination read (and, with a signer, written) through `collab`. + pub fn new(client: &'a PlatformClient, collab: Collab<'a>) -> Self { + Self { collab, client } + } +} + +impl Chain for CollabChain<'_> { + async fn upstream_targets( + &self, + repo: &RepoRef, + kind: TargetKind, + n: u32, + ) -> Result> { + self.collab.upstream_targets(repo, kind, n).await + } + + async fn upstream_numbered( + &self, + repo: &RepoRef, + kind: TargetKind, + all: bool, + ) -> Result> { + self.collab.upstream_numbered(repo, kind, all).await + } + + async fn imported_targets(&self, repo: &RepoRef) -> Result> { + // through Collab: a private destination's provenance is sealed (§7) + self.collab.imported_targets(repo).await + } + + async fn current(&self, repo: &RepoRef, target: &Target) -> Result { + let log = self.collab.target_log(repo, &target.id).await?; + let code = log.state_code(); + let labels = match target.kind { + TargetKind::Issue => issue_state_v2(code, &log.events).labels, + TargetKind::Patch => pr_state_v2(code, None, &log.events, None, |_, _| false).labels, + }; + Ok(Current { code, labels }) + } + + async fn comments(&self, repo: &RepoRef, target_id: &str) -> Result> { + Ok(self + .collab + .comments(repo, target_id) + .await? + .into_iter() + .map(|c| Written { + author: c.author, + url: c.imported.map(|i| i.url), + }) + .collect()) + } + + async fn reviews(&self, repo: &RepoRef, patch_id: &str) -> Result> { + Ok(self + .collab + .reviews(repo, patch_id) + .await? + .into_iter() + .map(|r| Written { + author: r.reviewer, + url: r.imported.map(|i| i.url), + }) + .collect()) + } + + async fn members(&self, repo: &RepoRef) -> Result> { + Ok(forge_core::members::MemberReader::new(self.client) + .list(repo) + .await? + .into_iter() + .map(|m| m.identity_id) + .collect()) + } + + async fn labels(&self, repo: &RepoRef) -> Result> { + self.collab.labels(repo).await + } + + async fn releases(&self, repo: &RepoRef) -> Result> { + Ok(self.collab.releases(repo).await?.current) + } + + async fn merge_base( + &self, + repo: &RepoRef, + base_ref: &str, + opened_at: u64, + freshness: Freshness, + ) -> Result { + // A private repo's ref names are sealed: Collab reads (and caches) its updates. + if repo.visibility == Visibility::Private { + return self.collab.base_ref_tips(repo, base_ref, opened_at).await; + } + let core = self.client.fetch_contract(&repo.forge().core).await?; + forge_core::refs::read_merge_base( + self.client, + &core, + &repo.scope()?, + base_ref, + opened_at, + freshness, + ) + .await + } + + fn refs_changed(&self) { + self.collab.refs_changed(); + } + + async fn balance(&self, identity: &str) -> Option { + self.client.get_balance(identity).await.ok() + } + + async fn create_imported( + &self, + repo: &RepoRef, + what: ImportedTarget<'_>, + from: Provenance<'_>, + ) -> Result { + self.collab.create_imported(repo, what, from).await + } + + async fn comment( + &self, + repo: &RepoRef, + target_id: &str, + body: &str, + anchor: Option<&CommentAnchor>, + imported: &Imported, + ) -> Result { + self.collab + .comment(repo, target_id, body, anchor, Some(imported)) + .await + } + + async fn review( + &self, + repo: &RepoRef, + patch_id: &str, + commit_oid: &[u8], + body: &str, + imported: &Imported, + ) -> Result { + self.collab + .review( + repo, + patch_id, + Verdict::Comment, + commit_oid, + body, + None, + Some(imported), + ) + .await + } + + async fn post_event( + &self, + repo: &RepoRef, + target: &Target, + kind: EventKind, + value: &str, + ) -> Result { + self.collab + .post_event(repo, target, kind, Some(value), None) + .await + } + + async fn write_transition( + &self, + repo: &RepoRef, + target: &Target, + mv: &TransitionMove, + merge_oid: Option<&[u8]>, + ) -> Result { + self.collab + .write_transition(repo, target, mv, merge_oid) + .await + } + + async fn create_label( + &self, + repo: &RepoRef, + name: &str, + color: &str, + description: &str, + ) -> Result { + self.collab + .create_label(repo, name, color, description, false) + .await + } + + async fn create_release(&self, repo: &RepoRef, input: &ReleaseInput) -> Result { + self.collab.create_release(repo, input).await + } + + async fn sync_sealed_releases( + &self, + ledger: &mut Ledger<'_>, + repo: &RepoRef, + releases: &[SrcRelease], + storage: Option<&ReleaseStorage>, + ) -> anyhow::Result<()> { + use anyhow::Context as _; + let targets = storage + .map(ReleaseStorage::targets) + .transpose() + .context("opening the release storage")?; + let dest = CollabDest { + collab: &self.collab, + repo, + store: targets.as_ref().map(ReleaseTargets::store), + }; + let fetch = crate::assets::Https::default(); + crate::sealed_release::sync(ledger, &dest, releases, &fetch).await + } +} diff --git a/crates/forge-import/src/dest.rs b/crates/forge-import/src/dest.rs index e15367f15..667065cab 100644 --- a/crates/forge-import/src/dest.rs +++ b/crates/forge-import/src/dest.rs @@ -18,6 +18,7 @@ use forge_core::collab::v2::Collab; use forge_core::repo::credits_to_dash; use crate::budget::{Budget, CapExceeded}; +use crate::chain::CollabChain; use crate::gitsync::ProofRepo; use crate::model::SrcCollab; use crate::sealed_release::ReleaseStorage; @@ -344,21 +345,23 @@ pub async fn dry_collab<'a>( src: &SrcCollab, mirror: Option, release_storage: Option, + lanes: usize, ) -> Result> { // A private destination is read with the signer's keys (its documents are sealed). let collab = match signer { Some(s) => Collab::new(client, &s.identity, &s.bridge), None => Collab::reader(client), }; - let mut dry = Sink::new( - collab, + let dry = Sink::new( + CollabChain::new(client, collab), existing, Ledger::new(client, signer.map(Signer::id), true, Budget::new(None)), ) .with_mirror(mirror) - .with_release_storage(release_storage); + .with_release_storage(release_storage) + .with_lanes(lanes); dry.sync(src).await?; - Ok(dry.ledger) + Ok(dry.into_ledger()) } /// What of `src` is written to a destination the mirror identity holds `role` in, and the @@ -432,6 +435,8 @@ pub struct CollabSource<'s> { pub mirror: Option, /// Where a private destination's sealed release files and asset lists go. pub release_storage: Option, + /// Lanes for the items' dependent writes ([`crate::pipeline`]; 1: one write at a time). + pub lanes: usize, } /// Write the collaboration documents missing from `repo` with the run's ledger (taken from @@ -450,6 +455,7 @@ pub async fn write_collab<'a>( src, mirror, release_storage, + lanes, } = source; let mut ledger = outcome.ledger.take().expect("the write phase has a ledger"); let private = repo.visibility == Visibility::Private; @@ -457,15 +463,19 @@ pub async fn write_collab<'a>( for w in warnings { ledger.warn(w); } - let mut sink = Sink::new( - Collab::new(client, &signer.identity, &signer.bridge), + let sink = Sink::new( + CollabChain::new( + client, + Collab::new(client, &signer.identity, &signer.bridge), + ), Some(repo), ledger, ) .with_mirror(mirror) - .with_release_storage(release_storage); + .with_release_storage(release_storage) + .with_lanes(lanes); let result = sink.sync(&plan).await; - outcome.ledger = Some(sink.ledger); + outcome.ledger = Some(sink.into_ledger()); result } diff --git a/crates/forge-import/src/importer.rs b/crates/forge-import/src/importer.rs index 0412a4ee8..05aaefa7c 100644 --- a/crates/forge-import/src/importer.rs +++ b/crates/forge-import/src/importer.rs @@ -53,6 +53,9 @@ pub struct ImportConfig { pub network: NetworkTarget, /// The signing identity source; optional for a dry run. pub key: Option, + /// Lanes for each issue's and PR's dependent writes (comments, reviews, labels, state), + /// behind its create ([`crate::pipeline`]); 1 writes one document at a time. + pub concurrency: usize, } /// Run an import. Always returns a summary (a failure is its status and error; what was @@ -255,6 +258,7 @@ async fn run_inner<'a>( &priced, mirror.clone(), release_storage.clone(), + cfg.concurrency, ) .await?; let collab_estimate = dry.budget.spent(); @@ -362,6 +366,7 @@ async fn run_inner<'a>( src: &collab_src, mirror, release_storage, + lanes: cfg.concurrency, }; dest::write_collab(client, signer, role, repo, source, definitions, outcome).await?; diff --git a/crates/forge-import/src/lib.rs b/crates/forge-import/src/lib.rs index 7320fce17..66bb14541 100644 --- a/crates/forge-import/src/lib.rs +++ b/crates/forge-import/src/lib.rs @@ -6,11 +6,15 @@ //! * [`sink`] — diff the desired collaboration state against the chain and write only the //! difference, every write charged to the [`budget`] before it is signed. //! * [`sealed_release`] — releases into a private destination, sealed (private-repos.md §16). +//! * [`pipeline`] — the write phase's schedule: creates in order, each item's other writes in +//! parallel lanes. +//! * [`chain`] — what the sink reads from and writes to the destination. //! * [`gitsync`] — git data through the ordinary `git-remote-dash` push. //! * [`summary`] — the run summary (table and `--summary-json`). pub mod assets; pub mod budget; +pub mod chain; pub mod claim; pub mod dest; pub mod github; @@ -18,6 +22,7 @@ pub mod gitlab; pub mod gitsync; pub mod importer; pub mod model; +pub mod pipeline; pub mod sealed_release; pub mod sink; pub mod source; @@ -25,3 +30,6 @@ pub mod source_github; pub mod source_gitlab; pub mod state; pub mod summary; + +#[cfg(test)] +mod sink_tests; diff --git a/crates/forge-import/src/main.rs b/crates/forge-import/src/main.rs index bf563e6f8..ad1f98be6 100644 --- a/crates/forge-import/src/main.rs +++ b/crates/forge-import/src/main.rs @@ -82,6 +82,17 @@ struct RunArgs { #[arg(long, default_value_t = 0)] limit: usize, + /// Parallel lanes for each issue's and PR's comments, reviews, labels and state, written + /// behind its create (issues and PRs themselves are always created one at a time, in + /// upstream order). 1 writes one document at a time. + #[arg( + long, + value_name = "N", + default_value_t = forge_import::pipeline::DEFAULT_LANES, + value_parser = forge_import::pipeline::parse_lanes + )] + concurrency: usize, + /// Write the run summary as JSON here (the Mirror Action reads it). #[arg(long)] summary_json: Option, @@ -213,6 +224,7 @@ async fn run(cli: Cli) -> Result { limit: r.limit, network: target(&r.net)?, key: key(&r.net), + concurrency: r.concurrency, }; Ok(finish(&importer::run(&cfg).await, r.summary_json.as_ref())) } @@ -254,6 +266,17 @@ mod tests { assert_eq!(cli.source.as_deref(), Some("o/r")); assert_eq!(cli.run.max_spend, Some(0.05)); assert!(cli.command.is_none()); + assert_eq!(cli.run.concurrency, forge_import::pipeline::DEFAULT_LANES); + } + + #[test] + fn concurrency_is_bounded() { + let parse = |n: &str| Cli::try_parse_from(["forge-import", "o/r", "--concurrency", n]); + assert_eq!(parse("1").unwrap().run.concurrency, 1); + assert_eq!(parse("16").unwrap().run.concurrency, 16); + assert!(parse("0").is_err()); + assert!(parse("17").is_err()); + assert!(parse("x").is_err()); } #[test] diff --git a/crates/forge-import/src/pipeline.rs b/crates/forge-import/src/pipeline.rs new file mode 100644 index 000000000..2de921ab6 --- /dev/null +++ b/crates/forge-import/src/pipeline.rs @@ -0,0 +1,477 @@ +//! The pipelined write phase: issue and PR creates strictly in order, each item's dependent +//! writes (comments, reviews, labels, state) in bounded parallel lanes. +//! +//! **Why this split.** Forge numbers issues and PRs densely (forge-v2 §6.2: `number == +//! countOf(issue) + countOf(patch) + 1`), so the creates are one sequence in upstream order, +//! each confirmed before the next ([`run`]'s *head*). Everything else about an item names only +//! the item itself: a comment or review `refersTo` its target (plus `lockGate`, which reads the +//! target's own lock transitions), a label event its target, and a transition's `c1`–`c6` +//! rules read the same target's earlier transitions. None of them moves the count the next +//! create is numbered by. So once item N is confirmed on chain, its *job* (the rest of its +//! writes) can run while N+1… are created. +//! +//! **Order kept.** Each job runs its writes one at a time, in the order the sequential run +//! wrote them: comments (source order), reviews (source order), label events, transitions. +//! Readers order a thread's timeline by `$createdAt`, the state rules need a target's +//! transitions in order, and L-46 puts the state after the thread; one job per item keeps all +//! three. Jobs of different items run in up to `lanes` lanes at once. +//! +//! **Stopping.** The first error that is not about one item (the spend cap, a network error) +//! stops the head from creating more and every lane from starting another write +//! ([`Stop::check`]); writes already in flight finish, so what landed is counted and paid for +//! before the run reports. That error is the run's. +//! +//! `lanes == 1` is the sequential run: create, then that item's writes, then the next item. + +use std::collections::BTreeMap; +use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use futures::channel::mpsc; +use futures::{SinkExt as _, StreamExt as _}; + +/// The default number of lanes: the window git-remote-dash's chunk upload has run at on every +/// network since spike S0.1 (`forge_core::backends::platform::PIPELINE_WINDOW`). +pub const DEFAULT_LANES: usize = forge_core::backends::platform::PIPELINE_WINDOW; + +/// The most lanes a run may use. +/// +/// Every write signs with its own identity-contract nonce (the SDK's nonce cache hands them +/// out under a lock), and Drive accepts them out of order, but only within 24 of the highest +/// that has landed (`MAX_MISSING_IDENTITY_REVISIONS`). A write that stalls (dropped from a +/// mempool, waiting out a quorum rotation) while 24 later ones land can no longer land with +/// its nonce: forge-core's write loop then finds its document absent by id and signs it +/// again with a new nonce (`create_attempts`, up to three times), so a stall costs time, not +/// a document, and never lands one twice. More lanes make that likelier, and the process's DAPI +/// pacing (8 requests a second) caps the useful number near [`DEFAULT_LANES`] anyway. +pub const MAX_LANES: usize = 16; + +/// Parse a lane count (`--concurrency`): 1..=[`MAX_LANES`]. +pub fn parse_lanes(s: &str) -> Result { + match s.parse::() { + Ok(n) if (1..=MAX_LANES).contains(&n) => Ok(n), + _ => Err(format!("expected a number from 1 to {MAX_LANES}")), + } +} + +/// Items the head may place ahead of the lanes, per lane. A crash leaves at most this many +/// items created with their threads unwritten; the next run finds them on chain and finishes +/// them. +const QUEUE_PER_LANE: usize = 4; + +/// Lock `m`, recovering from a panic elsewhere (the state is plain data). +pub(crate) fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// What a lane gets once the run is stopping: it starts no new write. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Stopped; + +impl std::fmt::Display for Stopped { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("stopping: an earlier write failed") + } +} + +impl std::error::Error for Stopped {} + +/// The stop signal shared by the head and the lanes, and the error that raised it. +#[derive(Debug, Default)] +pub struct Stop { + stopped: AtomicBool, + first: Mutex>, +} + +impl Stop { + /// Whether the run is stopping. + pub fn is_stopped(&self) -> bool { + self.stopped.load(Ordering::SeqCst) + } + + /// Refuse to start another write once the run is stopping. + pub fn check(&self) -> Result<()> { + if self.is_stopped() { + return Err(Stopped.into()); + } + Ok(()) + } + + /// Stop the run with `e`; the first error is the one reported. + pub fn fail(&self, e: anyhow::Error) { + let mut first = lock(&self.first); + if first.is_none() && e.downcast_ref::().is_none() { + *first = Some(e); + } + self.stopped.store(true, Ordering::SeqCst); + } + + fn take(&self) -> Option { + lock(&self.first).take() + } +} + +/// Run `items` through `open` (the head: strictly one after another, in order) and each job it +/// returns through `finish` (up to `lanes` at once). `open` returning `None` means the item has +/// nothing more to do. See the module docs. +pub async fn run( + items: impl IntoIterator, + lanes: usize, + stop: &Stop, + mut open: O, + finish: F, +) -> Result<()> +where + O: FnMut(T) -> OFut, + OFut: Future>>, + F: Fn(J) -> FFut, + FFut: Future>, +{ + if lanes <= 1 { + for item in items { + if let Some(job) = open(item).await? { + finish(job).await?; + } + } + return Ok(()); + } + let (mut tx, rx) = mpsc::channel::(lanes * QUEUE_PER_LANE); + let head = async move { + for item in items { + if stop.is_stopped() { + break; + } + match open(item).await { + Ok(Some(job)) => { + if tx.send(job).await.is_err() { + break; + } + } + Ok(None) => {} + Err(e) => { + stop.fail(e); + break; + } + } + } + // Dropping the sender ends the lanes' stream once they drain the queue. + }; + let finish = &finish; + let tail = rx.for_each_concurrent(lanes, |job| async move { + // Once stopping, queued jobs are dropped unstarted; the next run finds their items + // on chain and finishes them. + if stop.is_stopped() { + return; + } + if let Err(e) = finish(job).await { + stop.fail(e); + } + }); + futures::join!(head, tail); + stop.take().map_or(Ok(()), Err) +} + +/// Progress of the write phase for the log: how many items the head has placed (created, found, +/// or skipped), how many are complete, how many in run order are all complete (and the upstream +/// number of the last of them: with GitHub's one sequence, every item up to it; GitLab's issues +/// and merge requests interleave by creation time), documents written and spend. +#[derive(Debug)] +pub struct Progress { + total: usize, + every: Duration, + state: Mutex, +} + +#[derive(Debug)] +struct ProgressState { + started: Instant, + last: Instant, + printed: bool, + placed: usize, + placed_upstream: u32, + complete: usize, + /// Items at or past `low` that completed, by index, with their upstream numbers. + ahead: BTreeMap, + /// Every item below this index is complete. + low: usize, + low_upstream: u32, +} + +impl Progress { + /// Progress over `total` items, said at most every `every`. + pub fn new(total: usize, every: Duration) -> Self { + let now = Instant::now(); + Self { + total, + every, + state: Mutex::new(ProgressState { + started: now, + last: now, + printed: false, + placed: 0, + placed_upstream: 0, + complete: 0, + ahead: BTreeMap::new(), + low: 0, + low_upstream: 0, + }), + } + } + + /// The head is done with an item (upstream number `upstream`): created it, found it, or + /// skipped it. + pub fn placed(&self, upstream: u32) { + let mut s = lock(&self.state); + s.placed += 1; + s.placed_upstream = s.placed_upstream.max(upstream); + } + + /// Item `index` has nothing more to write this run. + pub fn completed(&self, index: usize, upstream: u32) { + let mut s = lock(&self.state); + s.complete += 1; + s.ahead.insert(index, upstream); + while let Some(n) = { + let low = s.low; + s.ahead.remove(&low) + } { + s.low += 1; + s.low_upstream = n; + } + } + + /// The line to log now, when one is due (`last`: the phase is over, said once a line was). + /// `written` counts the documents written so far and `spent_dash` the spend; a dry run + /// (`spent_dash` `None`) counts the documents it would write. + pub fn line(&self, last: bool, written: u64, spent_dash: Option) -> Option { + let mut s = lock(&self.state); + let now = Instant::now(); + let due = if last { + s.printed + } else { + now.duration_since(s.last) >= self.every + }; + if !due { + return None; + } + s.last = now; + s.printed = true; + let elapsed = now.duration_since(s.started).as_secs(); + let (what, docs) = match spent_dash { + Some(dash) => ( + "placed", + format!("{written} documents written, {dash:.6} DASH spent"), + ), + None => ("priced", format!("{written} documents to write (dry run)")), + }; + Some(format!( + "forge-import: progress: {}/{} issues and PRs {what} (up to upstream #{}), {} \ + complete (all of the first {}, through upstream #{}); {docs}; {}m{:02}s", + s.placed, + self.total, + s.placed_upstream, + s.complete, + s.low, + s.low_upstream, + elapsed / 60, + elapsed % 60 + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::collections::BTreeSet; + use std::rc::Rc; + + /// What the scheduler did, in order: `open i` / `start i` / `end i`. + #[derive(Default)] + struct Trace { + events: Vec, + running: BTreeSet, + most: usize, + opening: bool, + } + + async fn pause(i: usize) { + // Jobs of different lengths, so lanes overtake one another. + tokio::time::sleep(Duration::from_millis(((i * 7) % 5 + 1) as u64 * 10)).await; + } + + #[tokio::test(start_paused = true)] + async fn the_head_is_ordered_and_the_lanes_are_bounded() { + let trace = Rc::new(RefCell::new(Trace::default())); + let stop = Stop::default(); + let (t1, t2) = (trace.clone(), trace.clone()); + run( + 0..40usize, + 4, + &stop, + move |i| { + let t = t1.clone(); + async move { + { + let mut t = t.borrow_mut(); + assert!(!t.opening, "one create at a time"); + t.opening = true; + t.events.push(format!("open {i}")); + } + pause(i + 1).await; + t.borrow_mut().opening = false; + // every third item has nothing more to do + Ok((i % 3 != 0).then_some(i)) + } + }, + move |i| { + let t = t2.clone(); + async move { + { + let mut t = t.borrow_mut(); + assert!( + t.events.contains(&format!("open {i}")), + "a job starts after its item is placed" + ); + assert!(t.running.insert(i), "one job per item"); + t.most = t.most.max(t.running.len()); + t.events.push(format!("start {i}")); + } + pause(i).await; + let mut t = t.borrow_mut(); + t.running.remove(&i); + t.events.push(format!("end {i}")); + Ok(()) + } + }, + ) + .await + .unwrap(); + let t = trace.borrow(); + let opens: Vec = t + .events + .iter() + .filter_map(|e| e.strip_prefix("open ").map(|n| n.parse().unwrap())) + .collect(); + assert_eq!(opens, (0..40).collect::>(), "creates in order"); + assert!( + t.most <= 4 && t.most > 1, + "bounded, and parallel: {}", + t.most + ); + let ends = t.events.iter().filter(|e| e.starts_with("end ")).count(); + assert_eq!(ends, (0..40).filter(|i| i % 3 != 0).count()); + } + + /// The first real error stops the head and every lane from starting more; jobs in flight + /// finish; that error is returned, not a lane's `Stopped`. + #[tokio::test(start_paused = true)] + async fn the_first_error_stops_the_run_and_in_flight_jobs_finish() { + let started = Rc::new(RefCell::new(Vec::new())); + let finished = Rc::new(RefCell::new(Vec::new())); + let stop = Stop::default(); + let (s1, f1) = (started.clone(), finished.clone()); + let stop_ref = &stop; + let err = run( + 0..100usize, + 3, + &stop, + |i| async move { Ok(Some(i)) }, + move |i| { + let (s, f) = (s1.clone(), f1.clone()); + async move { + stop_ref.check()?; + s.borrow_mut().push(i); + if i == 5 { + anyhow::bail!("--max-spend reached"); + } + pause(i).await; + f.borrow_mut().push(i); + Ok(()) + } + }, + ) + .await + .unwrap_err(); + assert_eq!(err.to_string(), "--max-spend reached"); + let started = started.borrow(); + let finished = finished.borrow(); + assert!(started.len() < 20, "stopped early: {started:?}"); + // Every job that started (but #5) finished: nothing was cut off mid-write. + let mut want: Vec = started.iter().copied().filter(|&i| i != 5).collect(); + let mut got = finished.clone(); + want.sort_unstable(); + got.sort_unstable(); + assert_eq!(want, got); + } + + #[tokio::test] + async fn one_lane_is_the_sequential_run() { + let log = RefCell::new(Vec::new()); + run( + 0..5usize, + 1, + &Stop::default(), + |i| { + log.borrow_mut().push(format!("open {i}")); + async move { Ok(Some(i)) } + }, + |i| { + log.borrow_mut().push(format!("finish {i}")); + async { Ok(()) } + }, + ) + .await + .unwrap(); + let want: Vec = (0..5) + .flat_map(|i| [format!("open {i}"), format!("finish {i}")]) + .collect(); + assert_eq!(*log.borrow(), want); + // The head's error is returned as it is. + let e = run( + 0..3usize, + 1, + &Stop::default(), + |i| async move { + anyhow::ensure!(i < 1, "boom"); + Ok(Some(i)) + }, + |_| async { Ok(()) }, + ) + .await + .unwrap_err(); + assert_eq!(e.to_string(), "boom"); + } + + #[test] + fn progress_says_how_far_every_item_is_complete() { + let p = Progress::new(5, Duration::ZERO); + for (i, n) in [10, 11, 13, 14, 15].into_iter().enumerate() { + p.placed(n); + if i != 1 { + p.completed(i, n); + } + } + let line = p.line(false, 42, Some(0.5)).unwrap(); + assert!( + line.contains("5/5 issues and PRs placed (up to upstream #15), 4 complete (all of the first 1, through upstream #10)"), + "{line}" + ); + assert!( + line.contains("42 documents written, 0.500000 DASH spent"), + "{line}" + ); + p.completed(1, 11); + let last = p.line(true, 50, None).unwrap(); + assert!(last.contains("the first 5, through upstream #15"), "{last}"); + assert!(last.contains("5/5 issues and PRs priced"), "{last}"); + assert!(last.contains("50 documents to write (dry run)"), "{last}"); + // A phase that never said anything says nothing at its end either. + let quiet = Progress::new(1, Duration::from_secs(3600)); + assert!(quiet.line(false, 0, Some(0.0)).is_none()); + assert!(quiet.line(true, 0, Some(0.0)).is_none()); + } +} diff --git a/crates/forge-import/src/sink.rs b/crates/forge-import/src/sink.rs index a755717f4..2cd23ccc9 100644 --- a/crates/forge-import/src/sink.rs +++ b/crates/forge-import/src/sink.rs @@ -17,36 +17,42 @@ //! `upstreamNumber`. An earlier item cannot be placed after later ones, so an incremental run //! that finds one missing refuses to start ([`Sink::check_order`]). //! +//! **Pipelined.** The creates stay one strict sequence; each item's other writes (its thread, +//! labels and state) run in up to `lanes` parallel lanes once the item is confirmed +//! ([`crate::pipeline`]). Every lane charges the same budget, and every write is admitted +//! under the ledger's lock ([`Budget::reserve`]). +//! //! A dry run walks the same path with every write replaced by a count and an estimate. use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; use anyhow::{Context, Result}; -use forge_core::collab::v2::{ - Collab, ImportedTarget, PatchInput, PrBase, Provenance, Target, TargetKind, -}; -use forge_core::collab::{ReleaseInput, Verdict}; +use forge_core::collab::v2::{ImportedTarget, PatchInput, PrBase, Provenance, Target, TargetKind}; +use forge_core::collab::ReleaseInput; use forge_core::history::Freshness; use forge_core::platform::PlatformClient; +use forge_core::repo::credits_to_dash; use forge_core::rules::v2::{ - issue_state_v2, next_transition, pr_state_v2, Actor, StateAction, TransitionMove, - TransitionTarget, Visibility, + next_transition, Actor, StateAction, TransitionMove, TransitionTarget, Visibility, }; use forge_core::rules::{EventKind, MergeBaseTips}; use forge_core::scope::RepoRef; use crate::budget::{collab_doc_credits, Budget, CollabDoc}; +use crate::chain::{Chain, Current}; use crate::gitsync::{ProofRepo, Unfetched}; use crate::model::{same_item, same_item_renamed, SrcCollab, SrcLabel, SrcRelease, SrcTarget}; -use crate::sealed_release::{CollabDest, ReleaseStorage, ReleaseTargets}; +use crate::pipeline::{lock, Progress, Stop}; +use crate::sealed_release::ReleaseStorage; use crate::summary::Counts; /// The run's accounting: budget, counts, warnings. pub struct Ledger<'a> { - /// `None` only in the unit tests' offline ledger ([`Ledger::offline`]). client: Option<&'a PlatformClient>, signer: Option, dry_run: bool, @@ -69,43 +75,43 @@ impl<'a> Ledger<'a> { dry_run: bool, budget: Budget, ) -> Self { - Self { - client: Some(client), - signer, - dry_run, - budget, - counts: Counts::default(), - warnings: Vec::new(), - refused: BTreeSet::new(), - } + Self::with_client(Some(client), signer, dry_run, budget) } - /// A ledger with no Platform client, for the unit tests: nothing it does reads the - /// network (no signer, so no balance to reconcile). + /// A ledger that reads no balance itself: its sink measures through its [`Chain`]. + pub fn detached(signer: Option, dry_run: bool, budget: Budget) -> Self { + Self::with_client(None, signer, dry_run, budget) + } + + /// A ledger with no Platform client and no signer, for the unit tests: nothing it does + /// reads the network. #[cfg(test)] pub(crate) fn offline(dry_run: bool) -> Self { + Self::detached(None, dry_run, Budget::new(None)) + } + + /// Whether this is a dry run (every write replaced by a count and an estimate). + pub(crate) fn dry_run(&self) -> bool { + self.dry_run + } + + fn with_client( + client: Option<&'a PlatformClient>, + signer: Option, + dry_run: bool, + budget: Budget, + ) -> Self { Self { - client: None, - signer: None, + client, + signer, dry_run, - budget: Budget::new(None), + budget, counts: Counts::default(), warnings: Vec::new(), refused: BTreeSet::new(), } } - /// The Platform client the destination is read through. - fn client(&self) -> Result<&'a PlatformClient> { - self.client - .context("this ledger is offline: it has no Platform client to read with") - } - - /// Whether this is a dry run (every write replaced by a count and an estimate). - pub(crate) fn dry_run(&self) -> bool { - self.dry_run - } - /// Record a warning (also logged). Redacted: warnings are published (the Action's job /// summary), and some quote URLs or errors from the source. pub fn warn(&mut self, msg: impl Into) { @@ -144,9 +150,10 @@ impl<'a> Ledger<'a> { self.warn(msg); } - /// One write: charged to the budget first (refused past the cap, before anything is - /// signed), counted, executed unless this is a dry run, then reconciled with the - /// measured balance so an estimate that ran low stops the NEXT write. + /// One write outside the pipeline (a sealed release: one at a time, before any issue or + /// PR): charged to the budget first (refused past the cap, before anything is signed), + /// counted, executed unless this is a dry run, then reconciled with the measured balance so + /// an estimate that ran low stops the NEXT write. pub async fn write( &mut self, what: String, @@ -173,41 +180,23 @@ impl<'a> Ledger<'a> { /// The signer's balance, read only when the per-write cost trace is on /// (`RUST_LOG=forge_import::cost=debug`): it costs a query per write. pub async fn traced_balance(&self) -> Option { - if !tracing::enabled!(target: "forge_import::cost", tracing::Level::DEBUG) { + if !cost_traced() { return None; } self.balance(self.signer.as_deref()?).await } - /// Log one write's estimate against its measured balance drop (calibration). Nodes can - /// answer from a height before the write landed, so the balance is re-read (up to ~10 s) - /// until it moves; every write costs something. + /// Log one write's estimate against its measured balance drop (calibration). pub async fn trace_cost(&self, what: &str, estimated: u64, before: Option) { - let Some(before) = before else { return }; - let mut after = self.traced_balance().await; - for _ in 0..10 { - if after.is_some_and(|a| a != before) { - break; - } - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - after = self.traced_balance().await; - } - if let Some(after) = after { - tracing::debug!( - target: "forge_import::cost", - what, - estimated, - measured = before.saturating_sub(after), - "write cost" - ); - } + trace_cost(what, estimated, before, || self.traced_balance()).await; } /// Pull the measured balance drop into the budget; whether the balance could be read. pub async fn reconcile(&mut self) -> bool { - if let (Some(signer), Some(client)) = (&self.signer, self.client) { + let mark = self.budget.mark(); + if let (Some(client), Some(signer)) = (self.client, &self.signer) { if let Ok(balance) = client.get_balance(signer).await { - self.budget.reconcile(balance); + self.budget.measured(balance, mark); return true; } } @@ -218,28 +207,60 @@ impl<'a> Ledger<'a> { pub async fn balance(&self, identity: &str) -> Option { self.client?.get_balance(identity).await.ok() } +} - fn is_mine(&self, author: &str) -> bool { - self.signer.as_deref().is_none_or(|s| s == author) +/// Whether the per-write cost trace is on (`RUST_LOG=forge_import::cost=debug`). It reads the +/// balance around each write, which writes in flight together would blur: a traced run +/// writes one at a time. +pub fn cost_traced() -> bool { + tracing::enabled!(target: "forge_import::cost", tracing::Level::DEBUG) +} + +/// Log one write's estimate against its measured balance drop (calibration). Nodes can answer +/// from a height before the write landed, so the balance is re-read (up to ~10 s, `read`) +/// until it moves; every write costs something. +async fn trace_cost(what: &str, estimated: u64, before: Option, read: F) +where + F: Fn() -> Fut, + Fut: Future>, +{ + let Some(before) = before else { return }; + let mut after = read().await; + for _ in 0..10 { + if after.is_some_and(|a| a != before) { + break; + } + tokio::time::sleep(Duration::from_secs(1)).await; + after = read().await; + } + if let Some(after) = after { + tracing::debug!( + target: "forge_import::cost", + what, + estimated, + measured = before.saturating_sub(after), + "write cost" + ); } } -/// The destination and its accounting. -pub struct Sink<'a> { - collab: Collab<'a>, - /// `None` only in a dry run whose destination does not exist yet. - repo: Option, - /// Budget, counts, warnings. - pub ledger: Ledger<'a>, +/// How often lanes in flight together read the balance for the spend guard: at most once in +/// this long across all of them (each read is a DAPI request). Their reservations cover the +/// writes that finished since ([`Budget::spent`]). One lane reads after every write. +const MEASURE_EVERY: Duration = Duration::from_secs(2); + +/// How often the write phase logs its progress. +const PROGRESS_EVERY: Duration = Duration::from_secs(30); + +/// What the sink caches while it runs (read once, or kept from what it wrote). +#[derive(Default)] +struct Caches { /// Every imported issue and PR in the destination, `(imported.url, target)`, whoever /// wrote it; loaded on the first miss of a point lookup. index: Option>, /// The destination's members (maintainers and writers), read once: an imported item by /// a member is an earlier mirror's copy; one by anyone else is a squatter. members: Option>, - /// The git data a merge commit's ancestry is checked in, when this run has any (see - /// [`Sink::merge_proof`]). - mirror: Option, /// Each destination PR's base as the readers fold it, by target `$id`. Filled wherever /// a patch is read. opened: BTreeMap, @@ -249,9 +270,10 @@ pub struct Sink<'a> { /// PRs (destination `$id`) whose base was not a branch on chain when they were mirrored: /// no merge into it ever counts (D-501), so a later run cannot prove them either. never_provable: BTreeSet, - /// Bases this write phase read from the network already. The process's synced copy of - /// the history may be the dry run's, from before this run's push: a base's first read - /// here asks the network (`Freshness::Now`), later ones reuse that copy. + /// Bases this write phase has read from the network. The process's synced copy of the + /// history may be the dry run's, from before this run's push: until a base's first read + /// here has come back, reads ask the network (`Freshness::Now`), later ones reuse that + /// copy. fresh_read: BTreeSet, /// Each source item looked up this run, by `(tk, source number)`: its copy, or `None` /// when it has none (yet). @@ -259,6 +281,39 @@ pub struct Sink<'a> { /// The first source item stored at a number other than its own ([`Sink::create`]): /// numbers diverge from there on, said once. diverged: bool, +} + +/// An item placed on chain (created, or found) whose other writes are still to do: a lane's +/// job ([`crate::pipeline`]). +struct Job<'s> { + index: usize, + t: &'s SrcTarget, + target: Target, + fresh: bool, +} + +/// The destination and its accounting. Its methods take `&self`: the pipelined write phase +/// runs several at once, over state behind locks that are never held across an await. +pub struct Sink<'a, C: Chain> { + chain: C, + /// `None` only in a dry run whose destination does not exist yet. + repo: Option, + /// Budget, counts, warnings. + ledger: Mutex>, + /// The signer (copied from the ledger: read on every item). + signer: Option, + dry_run: bool, + /// The git data a merge commit's ancestry is checked in, when this run has any (see + /// [`Sink::merge_proof`]). + mirror: Option, + caches: Mutex, + /// Lanes for the items' dependent writes (1: one write at a time). + lanes: usize, + stop: Stop, + /// When lanes last read the balance ([`MEASURE_EVERY`]). + measured_at: Mutex>, + /// Held by the lane waiting out a base's read-after-write lag ([`Sink::merge_proof`]). + lag_gate: futures::lock::Mutex<()>, /// Where a private destination's sealed release files and asset lists go (`None`: the /// storage policy names no storage of your own). release_storage: Option, @@ -330,16 +385,9 @@ fn state_after(thread: &Result<()>) -> bool { thread.as_ref().err().is_none_or(item_error) } -/// A target's state on chain: its state code (the sum of its transitions) and its labels. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -struct Current { - code: i64, - labels: BTreeSet, -} - /// The state code `t` has at the source: an issue open (0) or closed (1); a PR open (0), /// closed (1), merged (2), and a draft adds 8 unless merged. -fn wanted_code(t: &SrcTarget) -> i64 { +pub(crate) fn wanted_code(t: &SrcTarget) -> i64 { let closed = i64::from(t.closed || t.merged_oid.is_some()); match t.kind { TargetKind::Issue => closed, @@ -352,7 +400,12 @@ fn wanted_code(t: &SrcTarget) -> i64 { /// `to` (breadth first over the legal moves, [`next_transition`]); empty when it already is /// there, or cannot get there (a merged PR is terminal). A merge is only a step when `to` is /// merged. -fn state_path(target: TransitionTarget, from: i64, to: i64, number: u32) -> Vec { +pub(crate) fn state_path( + target: TransitionTarget, + from: i64, + to: i64, + number: u32, +) -> Vec { use std::collections::VecDeque; const ACTIONS: [StateAction; 5] = [ StateAction::Close, @@ -475,22 +528,22 @@ fn text_doc(text: &str) -> u64 { text.len() as u64 + 160 } -impl<'a> Sink<'a> { - /// A sink for `repo` (see [`Ledger`]). - pub fn new(collab: Collab<'a>, repo: Option, ledger: Ledger<'a>) -> Self { +impl<'a, C: Chain> Sink<'a, C> { + /// A sink for `repo` (see [`Ledger`]), writing one item at a time until + /// [`Self::with_lanes`]. + pub fn new(chain: C, repo: Option, ledger: Ledger<'a>) -> Self { Self { - collab, + chain, repo, - ledger, - index: None, - members: None, + signer: ledger.signer.clone(), + dry_run: ledger.dry_run, + ledger: Mutex::new(ledger), mirror: None, - opened: BTreeMap::new(), - lag_waited: BTreeSet::new(), - never_provable: BTreeSet::new(), - fresh_read: BTreeSet::new(), - known: BTreeMap::new(), - diverged: false, + caches: Mutex::new(Caches::default()), + lanes: 1, + stop: Stop::default(), + measured_at: Mutex::new(None), + lag_gate: futures::lock::Mutex::new(()), release_storage: None, } } @@ -509,12 +562,112 @@ impl<'a> Sink<'a> { self } + /// Run the items' dependent writes in up to `lanes` lanes (1..=[`crate::pipeline::MAX_LANES`]; + /// 1 when a per-write cost trace is on, this crate's or forge-core's + /// `DASH_FORGE_COST_TRACE=1`, which read the balance around each write). + #[must_use] + pub fn with_lanes(mut self, lanes: usize) -> Self { + self.lanes = if cost_traced() || forge_core::backends::platform::pipeline_window() == 1 { + 1 + } else { + lanes.clamp(1, crate::pipeline::MAX_LANES) + }; + self + } + + /// The ledger, once the sink is done. + pub fn into_ledger(self) -> Ledger<'a> { + self.ledger + .into_inner() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn ledger(&self) -> MutexGuard<'_, Ledger<'a>> { + lock(&self.ledger) + } + + fn caches(&self) -> MutexGuard<'_, Caches> { + lock(&self.caches) + } + + fn is_mine(&self, author: &str) -> bool { + self.signer.as_deref().is_none_or(|s| s == author) + } + + fn warn(&self, msg: impl Into) { + self.ledger().warn(msg); + } + + /// One write: charged to the budget first (refused past the cap, before anything is + /// signed; reserved while other lanes write), counted, executed unless this is a dry run, + /// then reconciled with the measured balance so an estimate that ran low stops a later + /// write. Refused once the run is stopping ([`Stop`]). + async fn write( + &self, + what: String, + credits: u64, + count: fn(&mut Counts), + f: F, + ) -> Result> + where + F: FnOnce() -> Fut, + Fut: Future>, + { + self.stop.check()?; + { + let mut ledger = self.ledger(); + if self.dry_run { + ledger.budget.charge(credits, what.clone())?; + } else { + ledger.budget.reserve(credits, what.clone())?; + } + count(&mut ledger.counts); + } + if self.dry_run { + return Ok(None); + } + let before = self.traced_balance().await; + let out = f().await; + self.ledger().budget.finish(credits); + self.measure(false).await; + let out = out.with_context(|| format!("writing {what}"))?; + trace_cost(&what, credits, before, || self.traced_balance()).await; + Ok(Some(out)) + } + + /// The signer's balance, when the per-write cost trace is on (lanes are 1 then). + async fn traced_balance(&self) -> Option { + if !cost_traced() { + return None; + } + self.chain.balance(self.signer.as_deref()?).await + } + + /// Pull the measured balance drop into the budget: after every write with one lane; with + /// more, at most every [`MEASURE_EVERY`] across them, unless `now`. + async fn measure(&self, now: bool) { + let Some(signer) = self.signer.as_deref() else { + return; + }; + if self.lanes > 1 && !now { + let mut at = lock(&self.measured_at); + if at.is_some_and(|t| t.elapsed() < MEASURE_EVERY) { + return; + } + *at = Some(Instant::now()); + } + let mark = self.ledger().budget.mark(); + if let Some(balance) = self.chain.balance(signer).await { + self.ledger().budget.measured(balance, mark); + } + } + /// The base tips a reader checks PR `target_id`'s merge against: the base it was opened /// against, at its `$createdAt` ([`pr_base_tips`], as `Collab::patch_view` folds it). A PR /// this run has not read was opened now (after this run's push). None without a /// destination (a dry run of a repo not created yet). async fn base_tips( - &mut self, + &self, target_id: &str, base_ref: &str, freshness: Freshness, @@ -522,40 +675,36 @@ impl<'a> Sink<'a> { let PrBase { ref_name: base_ref, opened_at, - } = self.opened.get(target_id).cloned().unwrap_or(PrBase { - ref_name: base_ref.to_string(), - opened_at: u64::MAX, - }); + } = self + .caches() + .opened + .get(target_id) + .cloned() + .unwrap_or(PrBase { + ref_name: base_ref.to_string(), + opened_at: u64::MAX, + }); let Some(repo) = self.repo.as_ref() else { return Ok(MergeBaseTips::default()); }; - // A private repo's ref names are sealed: Collab reads (and caches) its updates. - if repo.visibility == Visibility::Private { - return Ok(self - .collab - .base_ref_tips(repo, &base_ref, opened_at) - .await?); - } // One read of the repo's git history per process (`Synced`), shared by every PR of // the run; `Now` when this run's own push may not show yet, and for a base's first - // read in a write phase (the synced copy may predate this run's push). - let first_write_read = !self.ledger.dry_run && self.fresh_read.insert(base_ref.clone()); + // read in a write phase (the synced copy may predate this run's push). Lanes that ask + // before that first read has come back ask the network too. + let first_write_read = !self.dry_run && !self.caches().fresh_read.contains(&base_ref); let freshness = if first_write_read { Freshness::Now } else { freshness }; - let client = self.ledger.client()?; - let core = client.fetch_contract(&repo.forge().core).await?; - Ok(forge_core::refs::read_merge_base( - client, - &core, - &repo.scope()?, - &base_ref, - opened_at, - freshness, - ) - .await?) + let tips = self + .chain + .merge_base(repo, &base_ref, opened_at, freshness) + .await?; + if first_write_read { + self.caches().fresh_read.insert(base_ref); + } + Ok(tips) } /// The oid a merge event for `t` names so that every reader counts it (D-602), or `None` @@ -576,7 +725,7 @@ impl<'a> Sink<'a> { /// /// A run without `code` pushes nothing: the git data is the base branches fetched for the /// proof ([`crate::gitsync::fetch_proof_bases`]), and only tips already on chain count. - async fn merge_proof(&mut self, t: &SrcTarget, target_id: &str) -> Result>> { + async fn merge_proof(&self, t: &SrcTarget, target_id: &str) -> Result>> { let (Some(merged), Some(patch), Some(proof)) = (&t.merged_oid, &t.patch, self.mirror.as_ref()) else { @@ -592,16 +741,17 @@ impl<'a> Sink<'a> { let (dir, pushed) = (proof.dir.clone(), proof.pushed); let contains = |tips: &MergeBaseTips| chain_tip_containing(&dir, tips, &merged); let mut tips = self.base_tips(target_id, base, Freshness::Synced).await?; + let opened = self.caches().opened.get(target_id).cloned(); // A PR opened against a base that did not exist then has no tips, and no merge into // it ever counts (D-501); naming one would be re-posted, and paid for, every run. - let base_counts = tips.tip.is_some() || !self.opened.contains_key(target_id); + let base_counts = tips.tip.is_some() || opened.is_none(); // Never provable: the PR was opened against a base with no tip on chain (read from // chain), or this run pushes nothing and the base never had a tip, so the PR it just // created is opened against a base that is no branch (D-501). if !base_counts || (!pushed && tips.historical.is_empty()) { - self.never_provable.insert(target_id.to_string()); + self.caches().never_provable.insert(target_id.to_string()); } - if self.ledger.dry_run { + if self.dry_run { return Ok(contains(&tips) .or(local.filter(|_| base_counts)) .and_then(|tip| hex::decode(tip).ok())); @@ -612,22 +762,32 @@ impl<'a> Sink<'a> { // Only where the wait can help: a base readers count for this PR, and the same ref // the destination PR folds against (a PR retargeted at the source keeps its original // base on chain); and once per base per run. - let folded = self - .opened - .get(target_id) + let settled = |tips: &MergeBaseTips| { + contains(tips).is_some() || local.as_ref().is_none_or(|l| tips.contains(l)) + }; + if settled(&tips) { + return Ok(contains(&tips).and_then(|tip| hex::decode(tip).ok())); + } + // One lane waits at a time. A lane that finds another waiting reads, once that one is + // done, the copy it brought up to date (`Synced`), as the next PR of a sequential run + // would, instead of settling for what it read before the wait. + let _gate = self.lag_gate.lock().await; + tips = self.base_tips(target_id, base, Freshness::Synced).await?; + let folded = opened + .as_ref() .map_or(base.as_str(), |b| b.ref_name.as_str()); - let may_lag = base_counts && folded == base && !self.lag_waited.contains(base); + let may_lag = + base_counts && folded == base && !self.caches().lag_waited.contains(base.as_str()); let waits = if may_lag { BASE_LAG_WAITS } else { &[] }; for wait in waits { - let seen = local.as_ref().is_none_or(|l| tips.contains(l)); - if contains(&tips).is_some() || seen { + if settled(&tips) { break; } // Marked only when a wait happens: a PR whose merge the mirror lacks (no `local`) // must not use up the one wait a later PR on the same base may need. - self.lag_waited.insert(base.clone()); - tokio::time::sleep(std::time::Duration::from_millis(*wait)).await; - self.collab.refs_changed(); + self.caches().lag_waited.insert(base.clone()); + tokio::time::sleep(Duration::from_millis(*wait)).await; + self.chain.refs_changed(); tips = self.base_tips(target_id, base, Freshness::Now).await?; } Ok(contains(&tips).and_then(|tip| hex::decode(tip).ok())) @@ -636,31 +796,31 @@ impl<'a> Sink<'a> { /// Whether `author` may have mirrored an item: the signer, or a member of the /// destination. Issues and PRs are ungated, so anyone else's `imported` is a claim, not a /// copy. - async fn trusted(&mut self, author: &str) -> Result { - if self.ledger.is_mine(author) { + async fn trusted(&self, author: &str) -> Result { + if self.is_mine(author) { return Ok(true); } - if self.members.is_none() { - let members = match &self.repo { - Some(repo) => forge_core::members::MemberReader::new(self.ledger.client()?) - .list(repo) - .await - .context("reading the destination's members")? - .into_iter() - .map(|m| m.identity_id) - .collect(), - None => BTreeSet::new(), - }; - self.members = Some(members); + if let Some(members) = &self.caches().members { + return Ok(members.contains(author)); } - Ok(self.members.as_ref().is_some_and(|m| m.contains(author))) + let members = match &self.repo { + Some(repo) => self + .chain + .members(repo) + .await + .context("reading the destination's members")?, + None => BTreeSet::new(), + }; + let trusted = members.contains(author); + self.caches().members = Some(members); + Ok(trusted) } /// Whether the document `target` (key `url`) is the mirrored copy of `t`: the same /// source item, written by the signer (a renamed source repo still matches) or by a /// member of the destination (exact repository only). - async fn is_copy(&mut self, url: &str, target: &Target, t: &SrcTarget) -> Result { - let mine = self.ledger.is_mine(&target.author); + async fn is_copy(&self, url: &str, target: &Target, t: &SrcTarget) -> Result { + let mine = self.is_mine(&target.author); // Membership is read only when it can change the answer. let member = !mine && target.kind == t.kind @@ -676,7 +836,7 @@ impl<'a> Sink<'a> { } /// Mirror everything in `src`: label definitions, releases, then issues and PRs. - pub async fn sync(&mut self, src: &SrcCollab) -> Result<()> { + pub async fn sync(&self, src: &SrcCollab) -> Result<()> { if let Some(labels) = &src.labels { self.sync_labels(labels).await?; } @@ -689,16 +849,64 @@ impl<'a> Sink<'a> { } } self.check_order(src).await?; - for t in &src.targets { - self.sync_target(t).await?; + self.sync_targets(&src.targets).await + } + + /// Every issue and PR: created (or found) strictly in order, the rest of each item's + /// writes pipelined behind ([`crate::pipeline::run`]). + async fn sync_targets(&self, targets: &[SrcTarget]) -> Result<()> { + let progress = Progress::new(targets.len(), PROGRESS_EVERY); + let progress = &progress; + let result = crate::pipeline::run( + targets.iter().enumerate(), + self.lanes, + &self.stop, + |(index, t)| async move { + let job = self.open_target(index, t).await; + if job.is_ok() { + progress.placed(t.number); + } + if matches!(job, Ok(None)) { + progress.completed(index, t.number); + } + self.say(progress, false); + job + }, + |job: Job<'_>| async move { + let (index, number) = (job.index, job.t.number); + self.finish_target(job).await?; + progress.completed(index, number); + self.say(progress, false); + Ok(()) + }, + ) + .await; + if self.lanes > 1 && !self.dry_run { + // What the lanes wrote since their last read. + self.measure(true).await; + } + self.say(progress, true); + result + } + + /// Log the write phase's progress when a line is due. + fn say(&self, progress: &Progress, last: bool) { + let (written, spent) = { + let l = self.ledger(); + let c = &l.counts; + let written = c.issues + c.prs + c.comments + c.reviews + c.events + c.transitions; + (written, l.budget.spent()) + }; + let spent = (!self.dry_run).then(|| credits_to_dash(spent)); + if let Some(line) = progress.line(last, written, spent) { + eprintln!("{line}"); } - Ok(()) } /// The highest source number of `kind` the destination holds from a trusted writer (the /// signer or a member), or 0. A stranger can set any `upstreamNumber`, so theirs are /// skipped; one page usually settles it, else every one is read. - async fn held_upstream(&mut self, kind: TargetKind) -> Result { + async fn held_upstream(&self, kind: TargetKind) -> Result { let Some(repo) = self.repo.clone() else { return Ok(0); }; @@ -706,7 +914,7 @@ impl<'a> Sink<'a> { // every one. Not "a full page": unreadable rows are dropped before this sees them. for all in [false, true] { let rows = self - .collab + .chain .upstream_numbered(&repo, kind, all) .await .context("reading the destination's upstream numbers")?; @@ -739,7 +947,7 @@ impl<'a> Sink<'a> { /// The check is per kind: GitHub numbers issues and PRs in one sequence, but a new issue /// below the highest mirrored PR (or the reverse) is not caught here; it takes the next /// number, and its upstream number stays with it. - async fn check_order(&mut self, src: &SrcCollab) -> Result<()> { + async fn check_order(&self, src: &SrcCollab) -> Result<()> { for kind in [TargetKind::Issue, TargetKind::Patch] { let held = self.held_upstream(kind).await?; let mut below = Vec::new(); @@ -768,7 +976,7 @@ impl<'a> Sink<'a> { missing.len() - 1 ); } - self.ledger.warn(format!( + self.warn(format!( "{} upstream {noun}(s) from #{first} are mirrored after #{held}: Forge \ numbers densely, so they take the next numbers (each keeps its source number \ as its upstream number)", @@ -780,10 +988,10 @@ impl<'a> Sink<'a> { // --- labels and releases ----------------------------------------------------------- - async fn sync_labels(&mut self, labels: &[SrcLabel]) -> Result<()> { + async fn sync_labels(&self, labels: &[SrcLabel]) -> Result<()> { let existing: BTreeMap = match &self.repo { Some(repo) => self - .collab + .chain .labels(repo) .await .context("reading the destination's labels")? @@ -795,7 +1003,7 @@ impl<'a> Sink<'a> { .collect(), None => BTreeMap::new(), }; - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); // Two source labels that clip to the same name would rewrite one definition on every // run; the first wins. let mut seen = BTreeSet::new(); @@ -807,18 +1015,17 @@ impl<'a> Sink<'a> { CollabDoc::Label, (l.name.len() + l.color.len() + l.description.len() + 60) as u64, ); - self.ledger - .write( - format!("label {}", l.name), - credits, - |c| c.labels += 1, - || async move { - collab - .create_label(need(repo)?, &l.name, &l.color, &l.description, false) - .await - }, - ) - .await?; + self.write( + format!("label {}", l.name), + credits, + |c| c.labels += 1, + || async move { + chain + .create_label(need(repo)?, &l.name, &l.color, &l.description) + .await + }, + ) + .await?; } Ok(()) } @@ -838,17 +1045,14 @@ impl<'a> Sink<'a> { /// signatures and common platform builds first ([`crate::model::fit_assets`]), and its /// notes end with a line saying how many are not mirrored, linking the source release /// ([`crate::model::notes_with_footer`]), which readers show. - /// - /// A private destination's releases are sealed instead ([`crate::sealed_release`]). - async fn sync_releases(&mut self, releases: &[SrcRelease]) -> Result<()> { + async fn sync_releases(&self, releases: &[SrcRelease]) -> Result<()> { let mut known = crate::assets::Known::default(); let existing: BTreeMap = match &self.repo { Some(repo) => self - .collab + .chain .releases(repo) .await .context("reading the destination's releases")? - .current .into_iter() .map(|r| { known.add(&r.tag_name, &r.assets); @@ -860,14 +1064,14 @@ impl<'a> Sink<'a> { }; let fetch = crate::assets::Https::default(); let mut budget = crate::assets::RUN_BYTES; - let dry = self.ledger.dry_run; + let dry = self.dry_run; // Newest first (the source's order): the hashing budget goes to the newest releases. let mut prepared = Vec::with_capacity(releases.len()); for r in releases { // A tag the destination would refuse (over 63 bytes, or outside the git ref // grammar the contract enforces, `@{` included) skips that release, not the run. if let Err(e) = forge_core::collab::v2::check_tag_name(&r.tag_name) { - self.ledger.skip(format!("release not mirrored: {e}")); + self.ledger().skip(format!("release not mirrored: {e}")); continue; } let mut assets_in = r.assets.clone(); @@ -881,7 +1085,7 @@ impl<'a> Sink<'a> { ) .await { - self.ledger.warn(w); + self.warn(w); } if dry { // Priced, not fetched: an asset a real run would hash changes the release. @@ -903,10 +1107,13 @@ impl<'a> Sink<'a> { // Counted and warned for the releases this run writes (a re-run that finds them // unchanged says nothing again). let unhashed = assets_in.iter().filter(|a| a.sha256.is_empty()).count(); - self.ledger.counts.assets_omitted += dropped as u64; - self.ledger.counts.assets_unhashed += unhashed as u64; + { + let mut ledger = self.ledger(); + ledger.counts.assets_omitted += dropped as u64; + ledger.counts.assets_unhashed += unhashed as u64; + } if dropped > 0 { - self.ledger.warn(format!( + self.warn(format!( "release {}: {dropped} of its {total} assets do not fit the 4096 bytes a \ release lists; left out (kept first: checksum files, signatures, the \ common platform builds; its notes link the source release)", @@ -925,110 +1132,138 @@ impl<'a> Sink<'a> { assets: assets_in, ..ReleaseInput::default() }; - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); let input = &input; let written = self - .ledger .write( format!("release {}", r.tag_name), credits, |c| c.releases += 1, - || async move { collab.create_release(need(repo)?, input).await }, + || async move { chain.create_release(need(repo)?, input).await }, ) .await; if let Err(e) = written { let item = item_error(&e); - self.ledger.refused_release(&r.tag_name, e, item)?; + self.ledger().refused_release(&r.tag_name, e, item)?; } } Ok(()) } /// A private destination's releases, sealed ([`crate::sealed_release`]), their files and - /// asset lists stored on the run's release storage. - async fn sync_sealed_releases(&mut self, releases: &[SrcRelease]) -> Result<()> { + /// asset lists stored on the run's release storage. They are written before any issue or + /// PR, one at a time, so the ledger is taken out of its lock for them and put back after. + async fn sync_sealed_releases(&self, releases: &[SrcRelease]) -> Result<()> { let Some(repo) = self.repo.as_ref() else { return Ok(()); }; - let targets = self - .release_storage - .as_ref() - .map(ReleaseStorage::targets) - .transpose() - .context("opening the release storage")?; - let dest = CollabDest { - collab: &self.collab, - repo, - store: targets.as_ref().map(ReleaseTargets::store), - }; - let fetch = crate::assets::Https::default(); - crate::sealed_release::sync(&mut self.ledger, &dest, releases, &fetch).await + let placeholder = Ledger::detached(self.signer.clone(), self.dry_run, Budget::new(None)); + let mut ledger = std::mem::replace(&mut *self.ledger(), placeholder); + let result = self + .chain + .sync_sealed_releases(&mut ledger, repo, releases, self.release_storage.as_ref()) + .await; + *self.ledger() = ledger; + result } // --- issues and pull requests -------------------------------------------------------- - /// Mirror one issue or PR. An error that only concerns this item (the destination - /// refused its content) skips it with a warning instead of failing the run, so one bad - /// item cannot stop every future run; spend-cap and network errors still stop the run. - async fn sync_target(&mut self, t: &SrcTarget) -> Result<()> { - match self.sync_target_inner(t).await { - Err(e) if item_error(&e) => { - self.ledger - .skip(format!("{} not mirrored this run: {e:#}", t.imported.url)); - // Only a refusal of the item's own content is final for it; a state move a - // concurrent change beat (a `c*` rule) or a taken number is retried next run. - if content_error(&e) { - self.ledger.refused.insert(key_of(t)); - } - Ok(()) - } - other => other, + /// An error that only concerns item `t` (the destination refused its content) skips it + /// with a warning instead of failing the run, so one bad item cannot stop every future + /// run; spend-cap and network errors still stop the run. + fn item_skipped(&self, t: &SrcTarget, e: anyhow::Error) -> Result<()> { + if !item_error(&e) { + return Err(e); + } + let mut ledger = self.ledger(); + ledger.skip(format!("{} not mirrored this run: {e:#}", t.imported.url)); + // Only a refusal of the item's own content is final for it; a state move a + // concurrent change beat (a `c*` rule) or a taken number is retried next run. + if content_error(&e) { + ledger.refused.insert(key_of(t)); } + Ok(()) } - async fn sync_target_inner(&mut self, t: &SrcTarget) -> Result<()> { + /// The head's part of item `t` (number `index` in the run): find its copy, or create it at + /// the dense next number. `None` when nothing more is written for it. + async fn open_target<'s>(&self, index: usize, t: &'s SrcTarget) -> Result>> { + match self.open_target_inner(index, t).await { + Err(e) => self.item_skipped(t, e).map(|()| None), + ok => ok, + } + } + + async fn open_target_inner<'s>( + &self, + index: usize, + t: &'s SrcTarget, + ) -> Result>> { let noun = t.kind.noun(); let (target, fresh) = if let Some(target) = self.existing(t).await? { (target, false) } else { let target = self.create(t, noun).await?; - if let Some(idx) = &mut self.index { + let mut caches = self.caches(); + if let Some(idx) = &mut caches.index { idx.push((t.imported.url.clone(), target.clone())); } - self.known.insert(key_of(t), Some(target.clone())); + caches.known.insert(key_of(t), Some(target.clone())); (target, true) }; - if !fresh && !self.ledger.is_mine(&target.author) { + if !fresh && !self.is_mine(&target.author) { // A member mirrored this item (an earlier mirror identity, or a second mirror): // writing to it, or recreating it at a new number, would duplicate it. (A // non-member's claim never gets here: it is a squatter, and the item is created // at another number.) Settled: the member's copy is permanent, so it does not hold // the sync state (a mirror identity rotated would otherwise hold it for good). - self.ledger.warn(format!( + self.warn(format!( "{noun} #{} is already mirrored by {} (as #{}); not mirrored again", t.number, target.author, target.number )); - return Ok(()); + return Ok(None); } - let current = if fresh { + Ok(Some(Job { + index, + t, + target, + fresh, + })) + } + + /// A lane's part of an item: its thread, then its state. + async fn finish_target(&self, job: Job<'_>) -> Result<()> { + let t = job.t; + match self.finish_target_inner(&job).await { + Err(e) => self.item_skipped(t, e), + ok => ok, + } + } + + async fn finish_target_inner(&self, job: &Job<'_>) -> Result<()> { + let Job { + t, target, fresh, .. + } = job; + let current = if *fresh { Current::default() } else { - self.current(&target).await? + self.current(target).await? }; // The thread first, then the state: readers order a target's timeline by `$createdAt`, // and at the source a close (or merge) comes after the comments that led to it (L-46). // A comment or review the destination refuses (an item error, which skips this item) // still lets the state be written, as it was before this order: a skip must not also // leave the item open. - let thread = self.sync_thread(t, &target, fresh).await; + let thread = self.sync_thread(t, target, *fresh).await; if state_after(&thread) { - self.sync_state(t, &target, ¤t).await?; + self.sync_state(t, target, ¤t).await?; } thread } /// An item's comments, then (a PR's) reviews. - async fn sync_thread(&mut self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { + async fn sync_thread(&self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { self.sync_comments(t, target, fresh).await?; if t.kind == TargetKind::Patch { self.sync_reviews(t, target, fresh).await?; @@ -1040,28 +1275,30 @@ impl<'a> Sink<'a> { /// each `issue` / `patch` of the repo that carries provenance, whoever wrote it. One /// complete read per kind, so an item is found wherever it landed; read only when its /// upstream number is held by a document that is not its copy. - async fn load_index(&mut self) -> Result<()> { - if self.index.is_some() { + async fn load_index(&self) -> Result<()> { + if self.caches().index.is_some() { return Ok(()); } let mut index = Vec::new(); + let mut opened = Vec::new(); if let Some(repo) = &self.repo { - // through Collab: a private destination's provenance is sealed (§7) let targets = self - .collab + .chain .imported_targets(repo) .await .context("reading the destination's issues and pull requests")?; for row in targets { if let Some(b) = row.base { - self.opened.insert(row.target.id.clone(), b); + opened.push((row.target.id.clone(), b)); } if let Some(i) = row.imported.filter(|i| !i.url.is_empty()) { index.push((i.url, row.target)); } } } - self.index = Some(index); + let mut caches = self.caches(); + caches.opened.extend(opened); + caches.index = Some(index); Ok(()) } @@ -1069,27 +1306,29 @@ impl<'a> Sink<'a> { /// looked up once per run. The `upstream (repoId, upstreamNumber)` index first (one read); /// the full `imported.url` index only when that number is held by something that is not /// its copy (a stranger can write any `upstreamNumber`). - async fn existing(&mut self, t: &SrcTarget) -> Result> { - if let Some(k) = self.known.get(&key_of(t)) { + async fn existing(&self, t: &SrcTarget) -> Result> { + if let Some(k) = self.caches().known.get(&key_of(t)) { return Ok(k.clone()); } let found = self.lookup(t).await?; - self.known.insert(key_of(t), found.clone()); + self.caches().known.insert(key_of(t), found.clone()); Ok(found) } - async fn lookup(&mut self, t: &SrcTarget) -> Result> { - if self.index.is_none() { + async fn lookup(&self, t: &SrcTarget) -> Result> { + if self.caches().index.is_none() { let Some(repo) = self.repo.as_ref() else { return Ok(None); }; - let rows = self.collab.upstream_targets(repo, t.kind, t.number).await?; + let rows = self.chain.upstream_targets(repo, t.kind, t.number).await?; if rows.is_empty() { return Ok(None); } for row in rows { if let Some(b) = &row.base { - self.opened.insert(row.target.id.clone(), b.clone()); + self.caches() + .opened + .insert(row.target.id.clone(), b.clone()); } if let Some(i) = row.imported.filter(|i| !i.url.is_empty()) { if self.is_copy(&i.url, &row.target, t).await? { @@ -1103,8 +1342,9 @@ impl<'a> Sink<'a> { } /// `t` in the full index (when loaded): the first entry that is its copy. - async fn indexed(&mut self, t: &SrcTarget) -> Result> { + async fn indexed(&self, t: &SrcTarget) -> Result> { let candidates: Vec<(String, Target)> = self + .caches() .index .as_ref() .map(|idx| { @@ -1128,18 +1368,18 @@ impl<'a> Sink<'a> { /// again when another create took the number first), with its provenance and its source /// number as `upstreamNumber`. In a dry run the returned target is a placeholder at the /// source number (nothing reads it). - async fn create(&mut self, t: &SrcTarget, noun: &str) -> Result { + async fn create(&self, t: &SrcTarget, noun: &str) -> Result { let placeholder = Target { kind: t.kind, id: String::new(), number: t.number, - author: self.ledger.signer.clone().unwrap_or_default(), + author: self.signer.clone().unwrap_or_default(), }; let credits = collab_doc_credits( CollabDoc::Target, text_doc(&t.title) + t.body.len() as u64 + t.imported.url.len() as u64 + 100, ); - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); let what = format!("{noun} #{}", t.number); let from = Provenance { imported: Some(&t.imported), @@ -1167,17 +1407,18 @@ impl<'a> Sink<'a> { TargetKind::Patch => |c| c.prs += 1, }; let out = self - .ledger .write(what, credits, count, || async move { - collab.create_imported(need(repo)?, doc, from).await + chain.create_imported(need(repo)?, doc, from).await }) .await?; let Some(created) = out else { return Ok(placeholder); }; - if created.number != t.number && !self.diverged { - self.diverged = true; - self.ledger.warn(diverged_note(noun, t, created.number)); + if created.number != t.number { + let first = !std::mem::replace(&mut self.caches().diverged, true); + if first { + self.warn(diverged_note(noun, t, created.number)); + } } Ok(Target { id: created.document_id, @@ -1188,29 +1429,23 @@ impl<'a> Sink<'a> { /// `target`'s state on chain: its state code (the sum of its transitions; "merged" is the /// chain fact, D-9) and its labels as every reader folds them. - async fn current(&mut self, target: &Target) -> Result { + async fn current(&self, target: &Target) -> Result { let repo = need(self.repo.as_ref())?; - let log = self.collab.target_log(repo, &target.id).await?; - let code = log.state_code(); - let labels = match target.kind { - TargetKind::Issue => issue_state_v2(code, &log.events).labels, - TargetKind::Patch => pr_state_v2(code, None, &log.events, None, |_, _| false).labels, - }; - Ok(Current { code, labels }) + Ok(self.chain.current(repo, target).await?) } /// The commit a merge transition for `t` names: a pushed base tip containing the source's /// merge commit when there is one ([`Sink::merge_proof`]), so readers find it on the base; /// else the source's merge commit itself. A merge is recorded either way (D-9: "merged" /// is what a member recorded); readers label one whose commit is not on the base. - async fn merge_oid(&mut self, t: &SrcTarget, target: &Target) -> Result> { + async fn merge_oid(&self, t: &SrcTarget, target: &Target) -> Result> { let upstream = t.merged_oid.clone().unwrap_or_default(); if let Some(tip) = self.merge_proof(t, &target.id).await? { return Ok(tip); } let base = t.patch.as_ref().map_or("?", |p| p.base_ref_name.as_str()); - self.ledger.counts.unproved_merges += 1; - let reason = if self.never_provable.contains(&target.id) { + let never_provable = self.caches().never_provable.contains(&target.id); + let reason = if never_provable { format!( "its base {base} was not a branch on chain when it was mirrored (forge-v2 §6, \ D-501: push the code before the issues and PRs)" @@ -1218,7 +1453,9 @@ impl<'a> Sink<'a> { } else { no_proof_reason(self.mirror.as_ref(), base) }; - self.ledger.warn(format!( + let mut ledger = self.ledger(); + ledger.counts.unproved_merges += 1; + ledger.warn(format!( "{} was merged; recorded as merged with the source's merge commit {}, which readers \ label as not found on the base: {reason}", t.imported.url, @@ -1230,12 +1467,7 @@ impl<'a> Sink<'a> { /// Bring `target` to `t`'s state: label events for the labels that differ, then the /// transitions ([`state_path`]) from its state code to the source's ([`wanted_code`]). /// Idempotent: a target already in that state takes nothing, and a merged one is final. - async fn sync_state( - &mut self, - t: &SrcTarget, - target: &Target, - current: &Current, - ) -> Result<()> { + async fn sync_state(&self, t: &SrcTarget, target: &Target, current: &Current) -> Result<()> { let moves = state_path( t.kind.transition_target(), current.code, @@ -1243,7 +1475,7 @@ impl<'a> Sink<'a> { target.number, ); if let Some(note) = merged_without_sha_note(t, current.code) { - self.ledger.warn(note); + self.warn(note); } let merges = moves .iter() @@ -1253,23 +1485,18 @@ impl<'a> Sink<'a> { } else { None }; - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); for (kind, value) in label_events(t, current) { let credits = collab_doc_credits(CollabDoc::Event, 120 + value.len() as u64); let what = format!("{kind:?} event on #{}", t.number); let value = value.as_str(); - self.ledger - .write( - what, - credits, - |c| c.events += 1, - || async move { - collab - .post_event(need(repo)?, target, kind, Some(value), None) - .await - }, - ) - .await?; + self.write( + what, + credits, + |c| c.events += 1, + || async move { chain.post_event(need(repo)?, target, kind, value).await }, + ) + .await?; } for mv in moves { let oid = (mv.kind == forge_core::rules::transition::PR_MERGE) @@ -1280,35 +1507,37 @@ impl<'a> Sink<'a> { TRANSITION_BYTES + oid.map_or(0, <[u8]>::len) as u64, ); let what = format!("kind-{} transition on #{}", mv.kind, t.number); - self.ledger - .write( - what, - credits, - |c| c.transitions += 1, - || async move { collab.write_transition(need(repo)?, target, &mv, oid).await }, - ) - .await?; + self.write( + what, + credits, + |c| c.transitions += 1, + || async move { chain.write_transition(need(repo)?, target, &mv, oid).await }, + ) + .await?; } Ok(()) } - async fn sync_comments(&mut self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { + /// The source URLs of the documents already on a thread written by the signer. + fn done_by_me(&self, written: Vec) -> BTreeSet { + written + .into_iter() + .filter(|w| self.is_mine(&w.author)) + .filter_map(|w| w.url) + .collect() + } + + async fn sync_comments(&self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { if t.comments.is_empty() { return Ok(()); } - let done: BTreeSet = if fresh { + let done = if fresh { BTreeSet::new() } else { let repo = need(self.repo.as_ref())?; - self.collab - .comments(repo, &target.id) - .await? - .into_iter() - .filter(|c| self.ledger.is_mine(&c.author)) - .filter_map(|c| c.imported.map(|i| i.url)) - .collect() + self.done_by_me(self.chain.comments(repo, &target.id).await?) }; - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); for c in t .comments .iter() @@ -1318,45 +1547,38 @@ impl<'a> Sink<'a> { CollabDoc::Comment, text_doc(&c.body) + c.imported.url.len() as u64, ); - self.ledger - .write( - format!("comment on #{}", t.number), - credits, - |n| n.comments += 1, - || async move { - collab - .comment( - need(repo)?, - &target.id, - &c.body, - c.anchor.as_ref(), - Some(&c.imported), - ) - .await - }, - ) - .await?; + self.write( + format!("comment on #{}", t.number), + credits, + |n| n.comments += 1, + || async move { + chain + .comment( + need(repo)?, + &target.id, + &c.body, + c.anchor.as_ref(), + &c.imported, + ) + .await + }, + ) + .await?; } Ok(()) } - async fn sync_reviews(&mut self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { + async fn sync_reviews(&self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { if t.reviews.is_empty() { return Ok(()); } - let done: BTreeSet = if fresh { + let done = if fresh { BTreeSet::new() } else { let repo = need(self.repo.as_ref())?; - self.collab - .reviews(repo, &target.id) - .await? - .into_iter() - .filter(|r| self.ledger.is_mine(&r.reviewer)) - .filter_map(|r| r.imported.map(|i| i.url)) - .collect() + self.done_by_me(self.chain.reviews(repo, &target.id).await?) }; - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); for r in t .reviews .iter() @@ -1366,29 +1588,20 @@ impl<'a> Sink<'a> { CollabDoc::Review, text_doc(&r.body) + r.imported.url.len() as u64 + 40, ); - self.ledger - .write( - format!("review on #{}", t.number), - credits, - |n| n.reviews += 1, - || async move { - collab - .review( - need(repo)?, - &target.id, - // Always a comment: the mirror identity is a member, and a - // member's approve / request-changes counts (§6); a source - // reviewer's verdict must not become one. It is in the body. - Verdict::Comment, - &r.commit_oid, - &r.body, - None, - Some(&r.imported), - ) - .await - }, - ) - .await?; + self.write( + format!("review on #{}", t.number), + credits, + |n| n.reviews += 1, + // Always a comment verdict ([`Chain::review`]): the mirror identity is a + // member, and a member's approve / request-changes counts (§6); a source + // reviewer's verdict must not become one. It is in the body. + || async move { + chain + .review(need(repo)?, &target.id, &r.commit_oid, &r.body, &r.imported) + .await + }, + ) + .await?; } Ok(()) } diff --git a/crates/forge-import/src/sink_tests.rs b/crates/forge-import/src/sink_tests.rs new file mode 100644 index 000000000..a3dc58ebc --- /dev/null +++ b/crates/forge-import/src/sink_tests.rs @@ -0,0 +1,805 @@ +//! The sink against an offline, recorded chain ([`Recorded`]): the documents it writes are +//! kept in memory in landing order, as Platform would hold them, with the rules the pipeline +//! must keep enforced as it writes: dense numbers (`number == count + 1`), a comment, review, +//! event or transition only on an issue or PR that exists (`refersTo`), and a transition only +//! from the state its target is in (`c1`–`c6`). Every write takes a different simulated time, +//! so lanes overtake one another, and a write can "crash the process": it lands, but the +//! client never hears so, and nothing started after it lands. +//! +//! The tests run the real [`Sink`] (diff, resume, pipeline, budget) over it: a pipelined +//! import writes exactly what the sequential one does, each item's documents in the same +//! order; a run crashed mid-pipeline and run again ends with every document once; the spend +//! cap holds with writes in flight together. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Mutex; +use std::time::Duration; + +use forge_core::collab::v2::{ + Created, ImportedRow, ImportedTarget, Provenance, Target, TargetKind, +}; +use forge_core::collab::{CommentAnchor, Imported, Label, Release, ReleaseInput}; +use forge_core::history::Freshness; +use forge_core::network::ForgeIds; +use forge_core::rules::v2::{TransitionMove, Visibility}; +use forge_core::rules::{EventKind, MergeBaseTips}; +use forge_core::scope::RepoRef; + +use crate::budget::Budget; +use crate::chain::{Chain, Current, Written}; +use crate::model::{SrcCollab, SrcComment, SrcPatch, SrcReview, SrcTarget}; +use crate::pipeline::lock; +use crate::sink::{Ledger, Sink}; + +const SIGNER: &str = "mirror"; +const START_BALANCE: u64 = 1_000_000_000_000_000; + +/// One issue or PR on the recorded chain. +#[derive(Debug, Clone)] +struct Item { + target: Target, + url: String, + upstream: u32, + code: i64, + labels: BTreeSet, + /// Its documents, in landing order: `c `, `r `, `e +label`, `t `. + log: Vec, +} + +#[derive(Debug, Default)] +struct State { + items: Vec, + /// Writes begun, in order. + begun: u64, + /// The write that crashes the process: it lands, its caller hears an error, and no write + /// begun after it lands. + crash_at: Option, + crashed: bool, + /// Writes in flight now, and the most at once; creates in flight; per target. + in_flight: usize, + most: usize, + creating: bool, + busy_targets: BTreeSet, + balance: u64, + /// A comment or review on a thread written while another of the same thread was in + /// flight, or a transition refused: must never happen. + violations: Vec, +} + +/// The recorded chain. +struct Recorded { + state: Mutex, + /// What each write really costs, as a share of its estimate (percent). + cost_pct: u64, + /// Source URLs whose document the chain refuses for its content (before it lands). + refuse: BTreeSet, +} + +impl Recorded { + fn new() -> Self { + Self { + state: Mutex::new(State { + balance: START_BALANCE, + ..State::default() + }), + cost_pct: 100, + refuse: BTreeSet::new(), + } + } + + fn st(&self) -> std::sync::MutexGuard<'_, State> { + lock(&self.state) + } + + /// One write on `target_id` (`None`: a create). Waits a simulated time that differs per + /// write, then lands `apply` unless the process crashed before it began. + async fn land( + &self, + target_id: Option<&str>, + apply: impl FnOnce(&mut State) -> forge_core::Result, + ) -> forge_core::Result { + let n = { + let mut st = self.st(); + if st.crashed { + return Err(forge_core::Error::Platform("connection lost".into())); + } + st.begun += 1; + st.in_flight += 1; + st.most = st.most.max(st.in_flight); + match target_id { + None => { + if std::mem::replace(&mut st.creating, true) { + st.violations.push("two creates in flight".into()); + } + } + Some(id) => { + if !st.busy_targets.insert(id.to_string()) { + st.violations.push(format!("two writes in flight on {id}")); + } + } + } + st.begun + }; + // 20–240 ms, different for neighbouring writes. + tokio::time::sleep(Duration::from_millis(20 + (n * 37 % 23) * 10)).await; + let mut st = self.st(); + st.in_flight -= 1; + match target_id { + None => st.creating = false, + Some(id) => { + st.busy_targets.remove(id); + } + } + // Landed: begun before the crash (the crashing write itself included). + let out = apply(&mut st); + if st.crash_at == Some(n) { + st.crashed = true; + } + if st.crashed { + return Err(forge_core::Error::Platform("connection lost".into())); + } + out + } + + fn charge(&self, st: &mut State, estimate: u64) { + st.balance = st.balance.saturating_sub(estimate * self.cost_pct / 100); + } + + fn row(item: &Item) -> ImportedRow { + ImportedRow { + target: item.target.clone(), + imported: Some(Imported { + url: item.url.clone(), + ..Imported::default() + }), + upstream_number: Some(item.upstream), + base: None, + } + } + + fn item<'s>(st: &'s mut State, id: &str) -> &'s mut Item { + st.items + .iter_mut() + .find(|i| i.target.id == id) + .unwrap_or_else(|| panic!("refersTo: no issue or PR {id}")) + } + + /// Every item's log, by upstream number, and the numbers they hold. + fn logs(&self) -> BTreeMap)> { + self.st() + .items + .iter() + .map(|i| (i.upstream, (i.target.number, i.log.clone()))) + .collect() + } +} + +/// What the recorded chain debits for a document (scaled by `cost_pct`): a little under the +/// sink's estimate for its kind (`crate::budget::CollabDoc::index_overhead` alone is 90M for +/// an issue or PR and at least 50M for the rest), as on the real networks. +fn price(kind: &str) -> u64 { + match kind { + "target" => 80_000_000, + _ => 45_000_000, + } +} + +// The reads answer at once; the trait is async for the network. +#[allow(clippy::unused_async_trait_impl)] +impl Chain for &Recorded { + async fn upstream_targets( + &self, + _: &RepoRef, + kind: TargetKind, + n: u32, + ) -> forge_core::Result> { + Ok(self + .st() + .items + .iter() + .filter(|i| i.target.kind == kind && i.upstream == n) + .map(Recorded::row) + .collect()) + } + + async fn upstream_numbered( + &self, + _: &RepoRef, + kind: TargetKind, + all: bool, + ) -> forge_core::Result> { + let mut rows: Vec = self + .st() + .items + .iter() + .filter(|i| i.target.kind == kind) + .map(Recorded::row) + .collect(); + rows.sort_by_key(|r| std::cmp::Reverse(r.upstream_number)); + if !all { + rows.truncate(100); + } + Ok(rows) + } + + async fn imported_targets(&self, _: &RepoRef) -> forge_core::Result> { + Ok(self.st().items.iter().map(Recorded::row).collect()) + } + + async fn current(&self, _: &RepoRef, target: &Target) -> forge_core::Result { + let mut st = self.st(); + let item = Recorded::item(&mut st, &target.id); + Ok(Current { + code: item.code, + labels: item.labels.clone(), + }) + } + + async fn comments(&self, _: &RepoRef, id: &str) -> forge_core::Result> { + let mut st = self.st(); + Ok(written(&Recorded::item(&mut st, id).log, "c ")) + } + + async fn reviews(&self, _: &RepoRef, id: &str) -> forge_core::Result> { + let mut st = self.st(); + Ok(written(&Recorded::item(&mut st, id).log, "r ")) + } + + async fn members(&self, _: &RepoRef) -> forge_core::Result> { + Ok(BTreeSet::new()) + } + + async fn labels(&self, _: &RepoRef) -> forge_core::Result> { + Ok(Vec::new()) + } + + async fn releases(&self, _: &RepoRef) -> forge_core::Result> { + Ok(Vec::new()) + } + + async fn merge_base( + &self, + _: &RepoRef, + _: &str, + _: u64, + _: Freshness, + ) -> forge_core::Result { + Ok(MergeBaseTips::default()) + } + + fn refs_changed(&self) {} + + async fn balance(&self, _: &str) -> Option { + Some(self.st().balance) + } + + async fn create_imported( + &self, + _: &RepoRef, + what: ImportedTarget<'_>, + from: Provenance<'_>, + ) -> forge_core::Result { + let kind = match what { + ImportedTarget::Issue { .. } => TargetKind::Issue, + ImportedTarget::Patch(_) => TargetKind::Patch, + }; + let url = from.imported.map(|i| i.url.clone()).unwrap_or_default(); + let upstream = from.upstream_number.unwrap_or_default(); + if self.refuse.contains(&url) { + return Err(forge_core::Error::RuleRefused { + document_type: kind.doc_type().into(), + rule: "hasTitle".into(), + detail: url, + }); + } + self.land(None, |st| { + // dense: the count of issues and PRs, plus one + let number = u32::try_from(st.items.len()).unwrap() + 1; + let id = format!("doc-{number}"); + st.items.push(Item { + target: Target { + kind, + id: id.clone(), + number, + author: SIGNER.into(), + }, + url, + upstream, + code: 0, + labels: BTreeSet::new(), + log: Vec::new(), + }); + self.charge(st, price("target")); + Ok(Created { + number, + document_id: id, + resumed: false, + draft_transition: None, + }) + }) + .await + } + + async fn comment( + &self, + _: &RepoRef, + target_id: &str, + _: &str, + _: Option<&CommentAnchor>, + imported: &Imported, + ) -> forge_core::Result { + self.append(target_id, format!("c {}", imported.url)).await + } + + async fn review( + &self, + _: &RepoRef, + patch_id: &str, + _: &[u8], + _: &str, + imported: &Imported, + ) -> forge_core::Result { + self.append(patch_id, format!("r {}", imported.url)).await + } + + async fn post_event( + &self, + _: &RepoRef, + target: &Target, + kind: EventKind, + value: &str, + ) -> forge_core::Result { + let add = kind == EventKind::LabelAdd; + let value = value.to_string(); + self.land(Some(&target.id), |st| { + self.charge(st, price("doc")); + let item = Recorded::item(st, &target.id); + if add { + item.labels.insert(value.clone()); + } else { + item.labels.remove(&value); + } + item.log + .push(format!("e {}{value}", if add { '+' } else { '-' })); + Ok(format!("e-{}", item.log.len())) + }) + .await + } + + async fn write_transition( + &self, + _: &RepoRef, + target: &Target, + mv: &TransitionMove, + _: Option<&[u8]>, + ) -> forge_core::Result { + let mv = *mv; + self.land(Some(&target.id), |st| { + // c1–c6: a move from the state the target is in, and nowhere else. + let code = Recorded::item(st, &target.id).code; + if code + mv.delta != mv.after { + let why = format!("kind {} from {code} on {}", mv.kind, target.id); + st.violations.push(format!("refused transition {why}")); + return Err(forge_core::Error::RuleRefused { + document_type: "transition".into(), + rule: "c1_closedAfter".into(), + detail: why, + }); + } + self.charge(st, price("doc")); + let item = Recorded::item(st, &target.id); + item.code = mv.after; + item.log.push(format!("t {}", mv.kind)); + Ok(format!("t-{}", item.log.len())) + }) + .await + } + + async fn create_label( + &self, + _: &RepoRef, + _: &str, + _: &str, + _: &str, + ) -> forge_core::Result { + unreachable!("no label definitions in these tests") + } + + async fn create_release(&self, _: &RepoRef, _: &ReleaseInput) -> forge_core::Result { + unreachable!("no releases in these tests") + } + + async fn sync_sealed_releases( + &self, + _: &mut Ledger<'_>, + _: &RepoRef, + _: &[crate::model::SrcRelease], + _: Option<&crate::sealed_release::ReleaseStorage>, + ) -> anyhow::Result<()> { + unreachable!("no releases in these tests") + } +} + +impl Recorded { + async fn append(&self, target_id: &str, entry: String) -> forge_core::Result { + if self.refuse.contains(&entry[2..]) { + return Err(forge_core::Error::Config("comment body too long".into())); + } + self.land(Some(target_id), |st| { + self.charge(st, price("doc")); + let item = Recorded::item(st, target_id); + item.log.push(entry); + Ok(format!("d-{}", item.log.len())) + }) + .await + } +} + +fn written(log: &[String], prefix: &str) -> Vec { + log.iter() + .filter_map(|e| e.strip_prefix(prefix)) + .map(|url| Written { + author: SIGNER.into(), + url: Some(url.to_string()), + }) + .collect() +} + +fn repo() -> RepoRef { + RepoRef { + forge: ForgeIds::test_forge(), + repo_id: "R".into(), + owner_id: SIGNER.into(), + name: "proj".into(), + visibility: Visibility::Public, + } +} + +fn imported(path: &str) -> Imported { + Imported { + author: "gh-user".into(), + created_at: 1, + url: format!("https://github.com/o/r/{path}"), + } +} + +/// A source of `n` items numbered 1..=n like GitHub's (issues and PRs in one sequence), with +/// threads of different lengths, labels, and every state: open, closed, merged, draft. +fn source(n: u32) -> SrcCollab { + let targets = (1..=n) + .map(|number| { + let pr = number % 3 != 0; + let path = if pr { "pull" } else { "issues" }; + let comments = (0..(number * 7 % 6)) + .map(|k| SrcComment { + body: format!("comment {k}"), + imported: imported(&format!("{path}/{number}#issuecomment-{k}")), + anchor: None, + }) + .collect(); + let reviews = if pr { + (0..(number % 4)) + .map(|k| SrcReview { + verdict: forge_core::collab::Verdict::Comment, + commit_oid: vec![1; 20], + body: format!("review {k}"), + imported: imported(&format!("pull/{number}#pullrequestreview-{k}")), + }) + .collect() + } else { + Vec::new() + }; + let labels = (0..(number % 3)).map(|k| format!("label-{k}")).collect(); + SrcTarget { + kind: if pr { + TargetKind::Patch + } else { + TargetKind::Issue + }, + number, + title: format!("item {number}"), + body: "b".into(), + imported: imported(&format!("{path}/{number}")), + closed: number % 2 == 0, + merged_oid: (pr && number % 4 == 0).then(|| vec![2; 20]), + merged_without_sha: false, + labels, + draft: pr && number % 5 == 1, + patch: pr.then(|| SrcPatch { + base_ref_name: "refs/heads/main".into(), + source_ref_name: None, + head_oid: vec![3; 20], + }), + comments, + reviews, + } + }) + .collect(); + SrcCollab { + targets, + ..SrcCollab::default() + } +} + +/// What each item's log must be after a complete import: its comments and reviews in source +/// order, then its label events, then its transitions (L-46), each once. +fn expected(src: &SrcCollab) -> BTreeMap> { + src.targets + .iter() + .map(|t| { + let mut log: Vec = t + .comments + .iter() + .map(|c| format!("c {}", c.imported.url)) + .collect(); + log.extend(t.reviews.iter().map(|r| format!("r {}", r.imported.url))); + log.extend(t.labels.iter().map(|l| format!("e +{l}"))); + let path = crate::sink::state_path( + t.kind.transition_target(), + 0, + crate::sink::wanted_code(t), + t.number, + ); + log.extend(path.iter().map(|mv| format!("t {}", mv.kind))); + (t.number, log) + }) + .collect() +} + +struct Run { + result: anyhow::Result<()>, + counts: crate::summary::Counts, + spent: u64, + refused: BTreeSet<(u8, u32)>, +} + +async fn import(chain: &Recorded, src: &SrcCollab, lanes: usize, cap: Option) -> Run { + let mut budget = Budget::new(cap); + budget.start(chain.st().balance); + let sink = Sink::new( + chain, + Some(repo()), + Ledger::detached(Some(SIGNER.into()), false, budget), + ) + .with_lanes(lanes); + let result = sink.sync(src).await; + let ledger = sink.into_ledger(); + Run { + result, + counts: ledger.counts, + spent: ledger.budget.spent(), + refused: ledger.refused, + } +} + +async fn dry(chain: &Recorded, src: &SrcCollab, lanes: usize) -> crate::summary::Counts { + let sink = Sink::new( + chain, + Some(repo()), + Ledger::detached(Some(SIGNER.into()), true, Budget::new(None)), + ) + .with_lanes(lanes); + sink.sync(src).await.unwrap(); + sink.into_ledger().counts +} + +fn docs(c: &crate::summary::Counts) -> u64 { + c.issues + c.prs + c.comments + c.reviews + c.events + c.transitions +} + +fn assert_complete(chain: &Recorded, src: &SrcCollab) { + let want = expected(src); + let got = chain.logs(); + assert_eq!(got.len(), want.len(), "every item once"); + for (upstream, (number, log)) in &got { + // A fresh mirror of a source with no gaps numbers identically, in upstream order. + assert_eq!(number, upstream, "dense numbers in upstream order"); + assert_eq!(log, &want[upstream], "item #{upstream}'s documents"); + } + let st = chain.st(); + assert!(st.violations.is_empty(), "{:?}", st.violations); +} + +/// A pipelined import writes exactly what a sequential one does: every item created in +/// upstream order at its number, each item's documents in the same order, the counts equal to +/// what landed, and the spend equal to the measured drop. And it really runs lanes at once. +#[tokio::test(start_paused = true)] +async fn a_pipelined_import_writes_what_a_sequential_one_does() { + let src = source(40); + let one = Recorded::new(); + let r1 = import(&one, &src, 1, None).await; + r1.result.unwrap(); + assert_complete(&one, &src); + assert!(one.st().most == 1, "one write at a time"); + + let eight = Recorded::new(); + let t0 = tokio::time::Instant::now(); + let r8 = import(&eight, &src, 8, None).await; + let pipelined = t0.elapsed(); + r8.result.unwrap(); + assert_complete(&eight, &src); + assert_eq!(one.logs(), eight.logs()); + let most = eight.st().most; + assert!((3..=9).contains(&most), "lanes plus the head: {most}"); + + // The progress and summary counts are what landed, and the spend what was paid. + assert_eq!(r8.counts, r1.counts); + let landed: u64 = eight.logs().values().map(|(_, l)| l.len() as u64).sum(); + assert_eq!(docs(&r8.counts), landed + src.targets.len() as u64); + // The spend reported: the estimates (the chain charged a little less), whatever the lanes. + assert_eq!(r8.spent, r1.spent); + assert!(r8.spent >= START_BALANCE - eight.st().balance); + + // Simulated time: the pipeline is several times faster. + let t0 = tokio::time::Instant::now(); + let again = Recorded::new(); + import(&again, &src, 1, None).await.result.unwrap(); + let sequential = t0.elapsed(); + assert!( + pipelined * 3 < sequential, + "pipelined {pipelined:?} vs sequential {sequential:?}" + ); +} + +/// A re-run over a complete mirror writes nothing, sequential or pipelined, and a dry run +/// prices the same at any lane count. +#[tokio::test(start_paused = true)] +async fn a_rerun_writes_nothing_and_dry_runs_agree() { + let src = source(25); + let chain = Recorded::new(); + let fresh_dry = dry(&chain, &src, 1).await; + assert_eq!(dry(&chain, &src, 8).await, fresh_dry); + let first = import(&chain, &src, 8, None).await; + first.result.unwrap(); + assert_eq!( + first.counts, fresh_dry, + "the dry run priced exactly what was written" + ); + let before = chain.logs(); + for lanes in [1, 8] { + let again = import(&chain, &src, lanes, None).await; + again.result.unwrap(); + assert_eq!(docs(&again.counts), 0); + assert_eq!(again.spent, 0); + } + assert_eq!(chain.logs(), before); + assert_eq!(docs(&dry(&chain, &src, 8).await), 0); +} + +/// The process dies mid-pipeline (a write lands but its caller never hears; nothing begun +/// after it lands). Run again, incrementally (the order check must not refuse it) and +/// pipelined: every item and document ends up on chain exactly once, in order, and the +/// resumed run writes exactly what a dry run between the two said was left. +#[tokio::test(start_paused = true)] +async fn a_run_crashed_mid_pipeline_resumes_to_the_same_mirror() { + let src = source(60); + for crash_at in [1, 7, 60, 131, 190] { + let chain = Recorded::new(); + chain.st().crash_at = Some(crash_at); + let crashed = import(&chain, &src, 8, None).await; + let err = crashed.result.unwrap_err(); + assert!(format!("{err:#}").contains("connection lost"), "{err:#}"); + let partial = chain.logs(); + assert!( + partial.len() < src.targets.len(), + "stopped early at {crash_at}" + ); + // Created items are a prefix in upstream order, at dense numbers. + for (n, (upstream, (number, _))) in (1u32..).zip(partial.iter()) { + assert_eq!((*upstream, *number), (n, n)); + } + { + let mut st = chain.st(); + st.crashed = false; + st.crash_at = None; + } + let left = dry(&chain, &src, 8).await; + let resumed_src = SrcCollab { + incremental: true, + ..src.clone() + }; + let resumed = import(&chain, &resumed_src, 8, None).await; + resumed.result.unwrap(); + assert_eq!( + resumed.counts, left, + "resumed exactly what was left ({crash_at})" + ); + assert_complete(&chain, &src); + } +} + +/// `--max-spend` with lanes in flight together: the run stops at the cap with the spend-cap +/// error, the measured drop never crosses the cap, every write it counted landed (in-flight +/// writes finish before it reports), and a re-run with a higher cap completes the mirror. +#[tokio::test(start_paused = true)] +async fn the_spend_cap_holds_with_lanes_in_flight() { + let src = source(50); + let chain = Recorded::new(); + let cap = 2_500_000_000; + let capped = import(&chain, &src, 8, Some(cap)).await; + let err = capped.result.unwrap_err(); + assert!( + err.downcast_ref::().is_some(), + "{err:#}" + ); + let paid = START_BALANCE - chain.st().balance; + assert!(paid <= cap, "paid {paid} over the cap {cap}"); + assert!(capped.spent <= cap); + let landed: u64 = chain.logs().values().map(|(_, l)| l.len() as u64).sum(); + assert_eq!( + docs(&capped.counts), + landed + chain.logs().len() as u64, + "the counts are what landed" + ); + import(&chain, &src, 8, None).await.result.unwrap(); + assert_complete(&chain, &src); +} + +/// Estimates that run low: the measured drop stops the run, pipelined too. What the lanes +/// already had in flight can take it past the cap by at most their own under-estimates. +#[tokio::test(start_paused = true)] +async fn a_low_estimate_is_caught_by_the_measured_drop_with_lanes() { + let src = source(50); + let mut chain = Recorded::new(); + chain.cost_pct = 150; + let cap = 2_500_000_000; + let capped = import(&chain, &src, 8, Some(cap)).await; + assert!(capped.result.is_err()); + let paid = START_BALANCE - chain.st().balance; + // Nine writes in flight at most (eight lanes and the head), each paid at most 1.5 times + // what the chain prices it at (a create: 80M), over its estimate. + let slack = 9 * 120_000_000; + assert!(paid <= cap + slack, "paid {paid}, cap {cap}"); + assert!(capped.spent >= paid, "the ledger reports what was paid"); +} + +/// Items the destination refuses for their content are skipped, in the head (a create) and in +/// a lane (a comment), and the run carries on: the refused create takes no number (the next +/// item takes it), and an item whose comment was refused still gets its labels and state, +/// but nothing after the refused comment. Both are recorded refused, and the result is the +/// same at one lane and at eight. +#[tokio::test(start_paused = true)] +async fn refused_items_are_skipped_and_the_run_carries_on() { + let src = source(30); + let comment = src.targets[9].comments[1].imported.url.clone(); + let create = src.targets[19].imported.url.clone(); + let want = expected(&src); + let mut results = Vec::new(); + for lanes in [1, 8] { + let mut chain = Recorded::new(); + chain.refuse = [comment.clone(), create.clone()].into(); + let run = import(&chain, &src, lanes, None).await; + run.result.unwrap(); + assert_eq!(run.counts.skipped, 2); + let refused: Vec = run.refused.iter().map(|(_, n)| *n).collect(); + assert_eq!(refused, [10, 20]); + let logs = chain.logs(); + assert!( + !logs.contains_key(&20), + "the refused create is not on chain" + ); + for (upstream, (number, log)) in &logs { + let n = if *upstream > 20 { + upstream - 1 + } else { + *upstream + }; + assert_eq!(*number, n, "dense: #{upstream} takes the next number"); + if *upstream != 10 { + assert_eq!(log, &want[upstream], "item #{upstream}"); + } + } + // #10: its first comment, then (past the refused one) its labels and state only. + let mut ten = vec![want[&10][0].clone()]; + ten.extend( + want[&10] + .iter() + .filter(|e| e.starts_with("e ") || e.starts_with("t ")) + .cloned(), + ); + assert_eq!(logs[&10].1, ten); + assert!(chain.st().violations.is_empty()); + results.push(logs); + } + assert_eq!(results[0], results[1]); +} From 5f1e692eeb897d058052512c7746525ddf9c5ef1 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 08:30:43 -0500 Subject: [PATCH 2/3] refactor(import): simplify the pipeline's helpers Counts::item_documents replaces three hand-written sums; open/finish wrap their item errors with or_else; one test helper builds the sink for the import and dry-run cases. No behaviour change. Co-Authored-By: Claude Opus 5.5 (1M context) --- crates/forge-import/src/pipeline.rs | 8 +++--- crates/forge-import/src/sink.rs | 39 ++++++++++++--------------- crates/forge-import/src/sink_tests.rs | 38 +++++++++++++------------- crates/forge-import/src/summary.rs | 6 +++++ 4 files changed, 44 insertions(+), 47 deletions(-) diff --git a/crates/forge-import/src/pipeline.rs b/crates/forge-import/src/pipeline.rs index 2de921ab6..bf05a020d 100644 --- a/crates/forge-import/src/pipeline.rs +++ b/crates/forge-import/src/pipeline.rs @@ -231,13 +231,11 @@ impl Progress { /// Item `index` has nothing more to write this run. pub fn completed(&self, index: usize, upstream: u32) { - let mut s = lock(&self.state); + let mut guard = lock(&self.state); + let s = &mut *guard; s.complete += 1; s.ahead.insert(index, upstream); - while let Some(n) = { - let low = s.low; - s.ahead.remove(&low) - } { + while let Some(n) = s.ahead.remove(&s.low) { s.low += 1; s.low_upstream = n; } diff --git a/crates/forge-import/src/sink.rs b/crates/forge-import/src/sink.rs index 2cd23ccc9..e1ccfcf96 100644 --- a/crates/forge-import/src/sink.rs +++ b/crates/forge-import/src/sink.rs @@ -740,6 +740,7 @@ impl<'a, C: Chain> Sink<'a, C> { let local = pushed_tip(proof, base, &merged); let (dir, pushed) = (proof.dir.clone(), proof.pushed); let contains = |tips: &MergeBaseTips| chain_tip_containing(&dir, tips, &merged); + let proven = |tips: &MergeBaseTips| contains(tips).and_then(|tip| hex::decode(tip).ok()); let mut tips = self.base_tips(target_id, base, Freshness::Synced).await?; let opened = self.caches().opened.get(target_id).cloned(); // A PR opened against a base that did not exist then has no tips, and no merge into @@ -766,7 +767,7 @@ impl<'a, C: Chain> Sink<'a, C> { contains(tips).is_some() || local.as_ref().is_none_or(|l| tips.contains(l)) }; if settled(&tips) { - return Ok(contains(&tips).and_then(|tip| hex::decode(tip).ok())); + return Ok(proven(&tips)); } // One lane waits at a time. A lane that finds another waiting reads, once that one is // done, the copy it brought up to date (`Synced`), as the next PR of a sequential run @@ -790,7 +791,7 @@ impl<'a, C: Chain> Sink<'a, C> { self.chain.refs_changed(); tips = self.base_tips(target_id, base, Freshness::Now).await?; } - Ok(contains(&tips).and_then(|tip| hex::decode(tip).ok())) + Ok(proven(&tips)) } /// Whether `author` may have mirrored an item: the signer, or a member of the @@ -863,19 +864,18 @@ impl<'a, C: Chain> Sink<'a, C> { &self.stop, |(index, t)| async move { let job = self.open_target(index, t).await; - if job.is_ok() { + if let Ok(job) = &job { progress.placed(t.number); - } - if matches!(job, Ok(None)) { - progress.completed(index, t.number); + if job.is_none() { + progress.completed(index, t.number); + } } self.say(progress, false); job }, |job: Job<'_>| async move { - let (index, number) = (job.index, job.t.number); - self.finish_target(job).await?; - progress.completed(index, number); + self.finish_target(&job).await?; + progress.completed(job.index, job.t.number); self.say(progress, false); Ok(()) }, @@ -893,9 +893,7 @@ impl<'a, C: Chain> Sink<'a, C> { fn say(&self, progress: &Progress, last: bool) { let (written, spent) = { let l = self.ledger(); - let c = &l.counts; - let written = c.issues + c.prs + c.comments + c.reviews + c.events + c.transitions; - (written, l.budget.spent()) + (l.counts.item_documents(), l.budget.spent()) }; let spent = (!self.dry_run).then(|| credits_to_dash(spent)); if let Some(line) = progress.line(last, written, spent) { @@ -1189,10 +1187,9 @@ impl<'a, C: Chain> Sink<'a, C> { /// The head's part of item `t` (number `index` in the run): find its copy, or create it at /// the dense next number. `None` when nothing more is written for it. async fn open_target<'s>(&self, index: usize, t: &'s SrcTarget) -> Result>> { - match self.open_target_inner(index, t).await { - Err(e) => self.item_skipped(t, e).map(|()| None), - ok => ok, - } + self.open_target_inner(index, t) + .await + .or_else(|e| self.item_skipped(t, e).map(|()| None)) } async fn open_target_inner<'s>( @@ -1233,12 +1230,10 @@ impl<'a, C: Chain> Sink<'a, C> { } /// A lane's part of an item: its thread, then its state. - async fn finish_target(&self, job: Job<'_>) -> Result<()> { - let t = job.t; - match self.finish_target_inner(&job).await { - Err(e) => self.item_skipped(t, e), - ok => ok, - } + async fn finish_target(&self, job: &Job<'_>) -> Result<()> { + self.finish_target_inner(job) + .await + .or_else(|e| self.item_skipped(job.t, e)) } async fn finish_target_inner(&self, job: &Job<'_>) -> Result<()> { diff --git a/crates/forge-import/src/sink_tests.rs b/crates/forge-import/src/sink_tests.rs index a3dc58ebc..0163214ea 100644 --- a/crates/forge-import/src/sink_tests.rs +++ b/crates/forge-import/src/sink_tests.rs @@ -553,15 +553,19 @@ struct Run { refused: BTreeSet<(u8, u32)>, } -async fn import(chain: &Recorded, src: &SrcCollab, lanes: usize, cap: Option) -> Run { - let mut budget = Budget::new(cap); - budget.start(chain.st().balance); - let sink = Sink::new( +fn sink(chain: &Recorded, dry_run: bool, budget: Budget, lanes: usize) -> Sink<'_, &Recorded> { + Sink::new( chain, Some(repo()), - Ledger::detached(Some(SIGNER.into()), false, budget), + Ledger::detached(Some(SIGNER.into()), dry_run, budget), ) - .with_lanes(lanes); + .with_lanes(lanes) +} + +async fn import(chain: &Recorded, src: &SrcCollab, lanes: usize, cap: Option) -> Run { + let mut budget = Budget::new(cap); + budget.start(chain.st().balance); + let sink = sink(chain, false, budget, lanes); let result = sink.sync(src).await; let ledger = sink.into_ledger(); Run { @@ -573,20 +577,11 @@ async fn import(chain: &Recorded, src: &SrcCollab, lanes: usize, cap: Option crate::summary::Counts { - let sink = Sink::new( - chain, - Some(repo()), - Ledger::detached(Some(SIGNER.into()), true, Budget::new(None)), - ) - .with_lanes(lanes); + let sink = sink(chain, true, Budget::new(None), lanes); sink.sync(src).await.unwrap(); sink.into_ledger().counts } -fn docs(c: &crate::summary::Counts) -> u64 { - c.issues + c.prs + c.comments + c.reviews + c.events + c.transitions -} - fn assert_complete(chain: &Recorded, src: &SrcCollab) { let want = expected(src); let got = chain.logs(); @@ -625,7 +620,10 @@ async fn a_pipelined_import_writes_what_a_sequential_one_does() { // The progress and summary counts are what landed, and the spend what was paid. assert_eq!(r8.counts, r1.counts); let landed: u64 = eight.logs().values().map(|(_, l)| l.len() as u64).sum(); - assert_eq!(docs(&r8.counts), landed + src.targets.len() as u64); + assert_eq!( + r8.counts.item_documents(), + landed + src.targets.len() as u64 + ); // The spend reported: the estimates (the chain charged a little less), whatever the lanes. assert_eq!(r8.spent, r1.spent); assert!(r8.spent >= START_BALANCE - eight.st().balance); @@ -659,11 +657,11 @@ async fn a_rerun_writes_nothing_and_dry_runs_agree() { for lanes in [1, 8] { let again = import(&chain, &src, lanes, None).await; again.result.unwrap(); - assert_eq!(docs(&again.counts), 0); + assert_eq!(again.counts.item_documents(), 0); assert_eq!(again.spent, 0); } assert_eq!(chain.logs(), before); - assert_eq!(docs(&dry(&chain, &src, 8).await), 0); + assert_eq!(dry(&chain, &src, 8).await.item_documents(), 0); } /// The process dies mid-pipeline (a write lands but its caller never hears; nothing begun @@ -727,7 +725,7 @@ async fn the_spend_cap_holds_with_lanes_in_flight() { assert!(capped.spent <= cap); let landed: u64 = chain.logs().values().map(|(_, l)| l.len() as u64).sum(); assert_eq!( - docs(&capped.counts), + capped.counts.item_documents(), landed + chain.logs().len() as u64, "the counts are what landed" ); diff --git a/crates/forge-import/src/summary.rs b/crates/forge-import/src/summary.rs index 0fe0a05a3..6d4179e81 100644 --- a/crates/forge-import/src/summary.rs +++ b/crates/forge-import/src/summary.rs @@ -62,6 +62,12 @@ impl Counts { self.packs += push.packs; self.pack_bytes += push.pack_bytes; } + + /// The issues and PRs, and the documents on them (comments, reviews, label events, + /// transitions): what the write phase's progress counts. + pub fn item_documents(&self) -> u64 { + self.issues + self.prs + self.comments + self.reviews + self.events + self.transitions + } } /// How the run ended. From 4d08bede049450c738de7017a122c3cbf0fa2189 Mon Sep 17 00:00:00 2001 From: pasta Date: Wed, 30 Sep 2026 11:48:46 -0500 Subject: [PATCH 3/3] docs(guides): document --concurrency and the pipelined import's timing The large-repositories section: creates in upstream order, each item's other writes in parallel lanes, the progress line, resuming an interrupted run, the spend cap with writes in flight, and the bonsia timing of dashpay/dash-network-deploy (122.6 min of writes one at a time, 38.1 min pipelined, same spend). Co-Authored-By: Claude Opus 5.5 (1M context) --- docs/guides/mirror-a-github-repo.md | 35 ++++++++++++++++++++++++++ docs/guides/mirror-a-gitlab-project.md | 2 +- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/docs/guides/mirror-a-github-repo.md b/docs/guides/mirror-a-github-repo.md index df80dea10..135d5f3d1 100644 --- a/docs/guides/mirror-a-github-repo.md +++ b/docs/guides/mirror-a-github-repo.md @@ -101,6 +101,7 @@ Useful flags (see `forge-import --help`): | `--include-label-definitions` | A private destination only: mirror the label definitions too. Their names, colours and descriptions are not encrypted, so they are left out by default; the labels set on issues and PRs are encrypted either way. | | `--limit ` | At most `n` issues and PRs (the oldest), for a cheap trial. Only those items and their comments are read, so a trial on a repository with thousands of issues takes a few requests. | | `--yes` | No confirmation prompt, for CI. | +| `--concurrency ` | How many issues and PRs have their comments, reviews, labels and state written at once (default 8, at most 16). Issues and PRs themselves are always created one at a time, in GitHub's order, so their numbers match. `1` writes one document at a time. See [large repositories](#large-repositories). | | `--summary-json ` | Write the run summary (counts, spend, key budget) as JSON. `forge-import` only. | | `--work-dir ` | Keep the bare git mirror between runs instead of a temporary directory. `forge-import` only. | | `--network`, `--devnet-name`, `--identity` | As for `dg`. `DASH_FORGE_KEY` may be a file path or an inline `dfk1:` key. | @@ -149,6 +150,40 @@ A signature or per-file checksum (`.asc`, `.sig`, `.minisig`, `.sha256`, ` **Not mirrored.** Edits to a title or body after the item was first mirrored, a PR's later retarget to another base, and a PR's later head moves (its `headOid` stays at the commit it was mirrored at; `refs/mirror/pull//head` follows the head while the PR is open). Reactions, milestones, assignees, projects and GitHub Discussions are not mirrored. +### Large repositories + +How the importer writes: +- **Issues and PRs** are created one at a time, in GitHub's order, each confirmed before the next, so their numbers follow GitHub's. +- **Everything else on an item** (its comments, reviews, labels and state) is written behind its creation, while later items are being created. `--concurrency` items (default 8) are written at once, and each item's own documents stay in their order: comments, then reviews, then labels and state. + +**How long it takes.** Timed on devnet bonsia (2026-09-30), importing the issues and PRs of `dashpay/dash-network-deploy` into two new repositories: 749 issues and PRs with 3,911 documents in all (1,097 comments, 1,301 reviews, 764 labels and state changes). + +| | `--concurrency 1` | `--concurrency 8` (the default) | +|---|---|---| +| Writing | 122.6 min (0.53 documents/s) | 38.1 min (1.71 documents/s) | +| Whole run, including about 10 minutes reading GitHub | 132 min | about 48 min | +| Spent (measured balance drop) | 3.164 DASH | 3.138 DASH | + +- **Speed:** writing was 3.2 times as fast; the cost was the same. +- **Interrupted run:** the pipelined run was killed partway through by accident and then run again. The two passes together wrote every document once, and a sample of 47 items (including every item that was mid-write at the kill) matched the other mirror exactly. +- **What limits it:** issues and PRs are still created one at a time, so a repository with many issues and PRs and few comments gains least. +- **dashpay/dash:** about 74,000 documents. It took 10 to 28 hours one document at a time. From these rates, a pipelined import of it should take about 7 to 9 hours. This is an estimate, not a measurement. + +**Progress.** Every 30 seconds the importer prints a progress line to stderr: + +``` +forge-import: progress: 412/749 issues and PRs placed (up to upstream #430), 405 complete (all of the first 398, through upstream #415); 2210 documents written, 1.874210 DASH spent; 9m30s +``` + +- **Complete:** an item whose documents are all written. +- **"all of the first N":** every item up to that point is complete. + +**Interrupted runs.** A run that stops (a crash, a network error, the spend cap) can leave a few dozen items created but not yet complete. Running it again finishes them: it reads each item's comments, reviews and state on chain and writes only what is missing, and it writes nothing twice. + +**Spend cap.** `--max-spend` still holds. Each write reserves its estimate against the cap before it is signed, counting the writes still in flight. When the cap is reached, no further write starts; the writes already in flight finish and are counted in the summary. + +**`--concurrency 1`.** This writes one document at a time, as earlier versions did. Use it when something else signs with the same identity at the same time, or when you measure each write's cost (`RUST_LOG=forge_import::cost=debug` selects it for you). + --- ## 2. The Forge Mirror Action diff --git a/docs/guides/mirror-a-gitlab-project.md b/docs/guides/mirror-a-gitlab-project.md index 17686ce93..040747835 100644 --- a/docs/guides/mirror-a-gitlab-project.md +++ b/docs/guides/mirror-a-gitlab-project.md @@ -59,7 +59,7 @@ Start with a dry run. It reads everything, compares it with what the destination forge-import gitlab.com/acme/widget --max-spend 0.5 --state ./widget.sync.json ``` -The flags are the same as for GitHub: `--repo`, `--sync`, `--max-spend`, `--state`, `--dry-run`, `--limit`, `--yes`, `--summary-json`, and the network flags. See the [GitHub guide's table](mirror-a-github-repo.md#1-first-import). In `--sync`, `prs` means merge requests (`mrs` works too). `dg import gitlab.com/acme/widget` is the same engine. +The flags are the same as for GitHub: `--repo`, `--sync`, `--max-spend`, `--state`, `--dry-run`, `--limit`, `--concurrency`, `--yes`, `--summary-json`, and the network flags. See the [GitHub guide's table](mirror-a-github-repo.md#1-first-import). In `--sync`, `prs` means merge requests (`mrs` works too). `dg import gitlab.com/acme/widget` is the same engine. **Re-running is safe and cheap.** What is already mirrored is decided on chain, by the GitLab URL each document records (`https://gitlab.com/acme/widget/-/issues/12`, `…/-/merge_requests/3`, and `…#note_` for a comment). A re-run writes only what is new and costs nothing when nothing changed. It is the same scheme as the GitHub importer.