diff --git a/Cargo.lock b/Cargo.lock index 1c2ee43cd..689f9fd10 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1900,6 +1900,7 @@ dependencies = [ "anyhow", "clap", "forge-core", + "futures", "hex", "reqwest", "serde", diff --git a/crates/dg/src/import.rs b/crates/dg/src/import.rs index d42d01cf2..0b622b68b 100644 --- a/crates/dg/src/import.rs +++ b/crates/dg/src/import.rs @@ -47,6 +47,15 @@ pub struct ImportArgs { /// Import at most this many issues and PRs (0 = all). #[arg(long, default_value_t = 0)] pub limit: usize, + /// Parallel lanes for each issue's and PR's comments, reviews, labels and state (issues + /// and PRs are always created one at a time, in upstream order). 1: one write at a time. + #[arg( + long, + value_name = "N", + default_value_t = forge_import::pipeline::DEFAULT_LANES, + value_parser = forge_import::pipeline::parse_lanes + )] + pub concurrency: usize, } fn max_spend(dash: Option) -> Result> { @@ -128,6 +137,7 @@ pub async fn import(ctx: &Ctx, a: &ImportArgs) -> Result<()> { limit: a.limit, network: ctx.target.clone(), key: ctx.identity_path.clone(), + concurrency: a.concurrency, }; report(ctx, &importer::run(&cfg).await) } diff --git a/crates/forge-import/Cargo.toml b/crates/forge-import/Cargo.toml index 094465592..00d7380c6 100644 --- a/crates/forge-import/Cargo.toml +++ b/crates/forge-import/Cargo.toml @@ -24,10 +24,13 @@ serde.workspace = true serde_json.workspace = true hex = "0.4" sha2 = "0.10" +futures = "0.3" reqwest = { version = "0.12", default-features = false, features = ["rustls-tls"] } [dev-dependencies] tempfile = "3" +# Paused time for the pipeline tests (`#[tokio::test(start_paused = true)]`). +tokio = { workspace = true, features = ["time", "test-util"] } [lints] workspace = true diff --git a/crates/forge-import/src/budget.rs b/crates/forge-import/src/budget.rs index f7d0b29e5..a8da6db1c 100644 --- a/crates/forge-import/src/budget.rs +++ b/crates/forge-import/src/budget.rs @@ -6,6 +6,16 @@ //! the estimates of the writes made and the identity's measured balance drop, so an //! estimate that runs low cannot carry the run past the cap for long: every //! [`Budget::reconcile`] pulls the measured figure in. +//! +//! **Writes in flight together** (the pipelined write phase, [`crate::pipeline`]) reserve +//! their estimate when admitted ([`Budget::reserve`]) and hold it until a balance read that +//! began after they finished has been folded in ([`Budget::finish`], [`Budget::mark`], +//! [`Budget::measured`]). Reads may come back in any order: each covers the writes finished +//! before it began, and the furthest such point folded in is what counts. +//! Admission is one synchronous step under the ledger's lock, so two writes can never both +//! pass on the same headroom, and "spent so far" never forgets a write the measured drop may +//! not show yet: it is the larger of the estimates and the measured drop plus every reserved +//! write's estimate. With one write at a time this is exactly the check above. use anyhow::{bail, Result}; @@ -126,6 +136,13 @@ pub struct Budget { estimated: u64, measured: u64, balance_start: Option, + /// Reserved writes still running ([`Self::reserve`]), credits. + in_flight: u64, + /// Every reserved write that finished, a running total ([`Self::finish`]), credits. + finished: u64, + /// How much of `finished` the balance reads folded in so far cover: the furthest + /// [`Self::mark`] handed back to [`Self::measured`]. + covered: u64, } impl Budget { @@ -136,6 +153,9 @@ impl Budget { estimated: 0, measured: 0, balance_start: None, + in_flight: 0, + finished: 0, + covered: 0, } } @@ -144,9 +164,13 @@ impl Budget { self.balance_start = Some(balance); } - /// What this run has spent, as far as it can tell (credits). + /// What this run has spent, as far as it can tell (credits): the estimates of every write + /// admitted, or the measured drop plus the reserved writes it may not show yet, whichever + /// is larger. pub fn spent(&self) -> u64 { - self.estimated.max(self.measured) + let unmeasured = self.finished.saturating_sub(self.covered); + let pending = self.in_flight.saturating_add(unmeasured); + self.estimated.max(self.measured.saturating_add(pending)) } /// Whether a write of `credits` would fit under the cap now. @@ -217,6 +241,37 @@ impl Budget { Ok(()) } + /// [`Self::charge`] for a write that may run beside others: its estimate stays reserved + /// on top of the measured drop until [`Self::finish`] and a balance read begun after it + /// ([`Self::mark`], [`Self::measured`]). + pub fn reserve(&mut self, credits: u64, what: impl Into) -> Result<()> { + self.charge(credits, what)?; + self.in_flight = self.in_flight.saturating_add(credits); + Ok(()) + } + + /// A reserved write of `credits` finished (landed or not): its estimate waits for the + /// next balance read. + pub fn finish(&mut self, credits: u64) { + let credits = credits.min(self.in_flight); + self.in_flight -= credits; + self.finished = self.finished.saturating_add(credits); + } + + /// Where a balance read starting now stands: it covers every reserved write finished so + /// far. Hand it to [`Self::measured`] with the read's result. + pub fn mark(&self) -> u64 { + self.finished + } + + /// Fold in a balance read that began at `mark`: the writes finished before it are in the + /// measured drop now, and no longer held on top of it. An older read folded in after a + /// newer one releases nothing more. + pub fn measured(&mut self, balance_now: u64, mark: u64) { + self.reconcile(balance_now); + self.covered = self.covered.max(mark); + } + /// Take back an admitted charge whose write stored nothing (a taken number). The /// measured drop still counts whatever fee was really paid. pub fn refund(&mut self, credits: u64) { @@ -271,6 +326,64 @@ mod tests { assert_eq!(b.spent(), 95); } + /// Writes in flight together: each is admitted against the others' reservations, and a + /// reservation stays on top of the measured drop until a balance read that began after the + /// write finished has been folded in. + #[test] + fn writes_in_flight_together_hold_their_reservations() { + let mut b = Budget::new(Some(100)); + b.start(1_000); + b.reserve(40, "a").unwrap(); + b.reserve(40, "b").unwrap(); + // A third would cross the cap on the reservations alone. + assert!(b.reserve(40, "c").is_err()); + assert_eq!(b.spent(), 80); + // `a` finishes; a balance read starts (covering `a`), and meanwhile `b` finishes. + b.finish(40); + let mark = b.mark(); + b.finish(40); + // The chain says 70 was spent: `a` really cost 70, and `b` may not show yet. + b.measured(930, mark); + assert_eq!(b.spent(), 110, "measured 70 plus b's reservation 40"); + assert!( + b.reserve(1, "d").is_err(), + "over the cap: nothing more is admitted" + ); + // The next read covers `b` too: only the measured drop is left. + let mark = b.mark(); + b.measured(920, mark); + assert_eq!(b.spent(), 80); + assert!(b.reserve(20, "e").is_ok()); + // One write at a time is the plain check. + let mut one = Budget::new(Some(100)); + one.start(1_000); + one.reserve(10, "x").unwrap(); + one.finish(10); + let mark = one.mark(); + one.measured(905, mark); + assert_eq!(one.spent(), 95); + } + + /// Two balance reads overlap and come back out of order: the older one must not release + /// a write that only the newer one's start came after, nor one that neither covers. + #[test] + fn balance_reads_folded_in_out_of_order_release_only_what_they_cover() { + let mut b = Budget::new(Some(1_000)); + b.start(10_000); + for w in [10, 15, 5] { + b.reserve(w, "w").unwrap(); + } + b.finish(10); + let r1 = b.mark(); // covers the 10 + b.finish(15); + let r2 = b.mark(); // covers the 10 and the 15 + b.finish(5); // neither read covers it + b.measured(9_975, r2); + b.measured(9_990, r1); + // Measured 25 (the larger drop), plus the 5 no read covers. + assert_eq!(b.spent(), 30); + } + #[test] fn the_plan_is_refused_whole_before_any_write() { let b = Budget::new(Some(50)); diff --git a/crates/forge-import/src/chain.rs b/crates/forge-import/src/chain.rs new file mode 100644 index 000000000..e04203bca --- /dev/null +++ b/crates/forge-import/src/chain.rs @@ -0,0 +1,375 @@ +//! What the [`crate::sink::Sink`] reads from and writes to the destination, as one trait: the +//! production [`CollabChain`] over forge-core's `Collab`, and an in-memory chain in the tests +//! (`crate::sink_tests`), so the importer's diff, resume and pipelining run offline. +//! +//! Every method is one of `Collab`'s (or the client's), narrowed to what the sink uses: a +//! target's state as the sink compares it ([`Current`]), and a thread's comments and reviews +//! as who wrote them and which source item each copies ([`Written`]). + +use std::collections::BTreeSet; + +use forge_core::collab::v2::{ + Collab, Created, ImportedRow, ImportedTarget, Provenance, Target, TargetKind, +}; +use forge_core::collab::{CommentAnchor, Imported, Label, Release, ReleaseInput, Verdict}; +use forge_core::history::Freshness; +use forge_core::platform::PlatformClient; +use forge_core::rules::v2::{issue_state_v2, pr_state_v2, TransitionMove, Visibility}; +use forge_core::rules::{EventKind, MergeBaseTips}; +use forge_core::scope::RepoRef; +use forge_core::Result; + +use crate::model::SrcRelease; +use crate::sealed_release::{CollabDest, ReleaseStorage, ReleaseTargets}; +use crate::sink::Ledger; + +/// A target's state on chain: its state code (the sum of its transitions) and its labels. +#[derive(Debug, Clone, Default, PartialEq, Eq)] +pub struct Current { + /// The state code. + pub code: i64, + /// The labels, as every reader folds the target's events. + pub labels: BTreeSet, +} + +/// A comment or review already on a target: who wrote it and which source item it copies. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Written { + /// Its author (a review's reviewer). + pub author: String, + /// Its `imported.url`, when it was imported. + pub url: Option, +} + +/// The destination operations the sink needs. Async methods on `&self`: the pipelined write +/// phase calls them from several lanes at once. +#[allow(async_fn_in_trait)] // used through generics only, never boxed or sent +pub trait Chain { + /// The `kind` documents of `repo` whose `upstreamNumber` is `n`. + async fn upstream_targets( + &self, + repo: &RepoRef, + kind: TargetKind, + n: u32, + ) -> Result>; + /// The `kind` documents carrying an `upstreamNumber`, highest first: one page, or `all`. + async fn upstream_numbered( + &self, + repo: &RepoRef, + kind: TargetKind, + all: bool, + ) -> Result>; + /// Every issue and PR of `repo` with its provenance. + async fn imported_targets(&self, repo: &RepoRef) -> Result>; + /// `target`'s state code and labels. + async fn current(&self, repo: &RepoRef, target: &Target) -> Result; + /// The comments on `target_id`. + async fn comments(&self, repo: &RepoRef, target_id: &str) -> Result>; + /// The reviews of `patch_id`. + async fn reviews(&self, repo: &RepoRef, patch_id: &str) -> Result>; + /// The destination's members (maintainers and writers). + async fn members(&self, repo: &RepoRef) -> Result>; + /// The label definitions. + async fn labels(&self, repo: &RepoRef) -> Result>; + /// The release of each live tag. + async fn releases(&self, repo: &RepoRef) -> Result>; + /// The tips a merge into `base_ref` (of a PR opened at `opened_at`) is checked against. + async fn merge_base( + &self, + repo: &RepoRef, + base_ref: &str, + opened_at: u64, + freshness: Freshness, + ) -> Result; + /// Forget cached ref updates (this process pushed since they were read). + fn refs_changed(&self); + /// `identity`'s balance now, when it can be read. + async fn balance(&self, identity: &str) -> Option; + + /// Create an imported issue or PR at the dense next number. + async fn create_imported( + &self, + repo: &RepoRef, + what: ImportedTarget<'_>, + from: Provenance<'_>, + ) -> Result; + /// Comment on `target_id`. + async fn comment( + &self, + repo: &RepoRef, + target_id: &str, + body: &str, + anchor: Option<&CommentAnchor>, + imported: &Imported, + ) -> Result; + /// Review `patch_id` (always a comment verdict: see [`crate::sink`]). + async fn review( + &self, + repo: &RepoRef, + patch_id: &str, + commit_oid: &[u8], + body: &str, + imported: &Imported, + ) -> Result; + /// A member label event on `target`. + async fn post_event( + &self, + repo: &RepoRef, + target: &Target, + kind: EventKind, + value: &str, + ) -> Result; + /// The state transition `mv` on `target`. + async fn write_transition( + &self, + repo: &RepoRef, + target: &Target, + mv: &TransitionMove, + merge_oid: Option<&[u8]>, + ) -> Result; + /// Define a label. + async fn create_label( + &self, + repo: &RepoRef, + name: &str, + color: &str, + description: &str, + ) -> Result; + /// Publish a release. + async fn create_release(&self, repo: &RepoRef, input: &ReleaseInput) -> Result; + /// A private destination's releases, sealed ([`crate::sealed_release::sync`]), their files + /// and asset lists stored on `storage`. + async fn sync_sealed_releases( + &self, + ledger: &mut Ledger<'_>, + repo: &RepoRef, + releases: &[SrcRelease], + storage: Option<&ReleaseStorage>, + ) -> anyhow::Result<()>; +} + +/// The destination on Dash Platform, through forge-core's `Collab`. +pub struct CollabChain<'a> { + collab: Collab<'a>, + client: &'a PlatformClient, +} + +impl<'a> CollabChain<'a> { + /// The destination read (and, with a signer, written) through `collab`. + pub fn new(client: &'a PlatformClient, collab: Collab<'a>) -> Self { + Self { collab, client } + } +} + +impl Chain for CollabChain<'_> { + async fn upstream_targets( + &self, + repo: &RepoRef, + kind: TargetKind, + n: u32, + ) -> Result> { + self.collab.upstream_targets(repo, kind, n).await + } + + async fn upstream_numbered( + &self, + repo: &RepoRef, + kind: TargetKind, + all: bool, + ) -> Result> { + self.collab.upstream_numbered(repo, kind, all).await + } + + async fn imported_targets(&self, repo: &RepoRef) -> Result> { + // through Collab: a private destination's provenance is sealed (§7) + self.collab.imported_targets(repo).await + } + + async fn current(&self, repo: &RepoRef, target: &Target) -> Result { + let log = self.collab.target_log(repo, &target.id).await?; + let code = log.state_code(); + let labels = match target.kind { + TargetKind::Issue => issue_state_v2(code, &log.events).labels, + TargetKind::Patch => pr_state_v2(code, None, &log.events, None, |_, _| false).labels, + }; + Ok(Current { code, labels }) + } + + async fn comments(&self, repo: &RepoRef, target_id: &str) -> Result> { + Ok(self + .collab + .comments(repo, target_id) + .await? + .into_iter() + .map(|c| Written { + author: c.author, + url: c.imported.map(|i| i.url), + }) + .collect()) + } + + async fn reviews(&self, repo: &RepoRef, patch_id: &str) -> Result> { + Ok(self + .collab + .reviews(repo, patch_id) + .await? + .into_iter() + .map(|r| Written { + author: r.reviewer, + url: r.imported.map(|i| i.url), + }) + .collect()) + } + + async fn members(&self, repo: &RepoRef) -> Result> { + Ok(forge_core::members::MemberReader::new(self.client) + .list(repo) + .await? + .into_iter() + .map(|m| m.identity_id) + .collect()) + } + + async fn labels(&self, repo: &RepoRef) -> Result> { + self.collab.labels(repo).await + } + + async fn releases(&self, repo: &RepoRef) -> Result> { + Ok(self.collab.releases(repo).await?.current) + } + + async fn merge_base( + &self, + repo: &RepoRef, + base_ref: &str, + opened_at: u64, + freshness: Freshness, + ) -> Result { + // A private repo's ref names are sealed: Collab reads (and caches) its updates. + if repo.visibility == Visibility::Private { + return self.collab.base_ref_tips(repo, base_ref, opened_at).await; + } + let core = self.client.fetch_contract(&repo.forge().core).await?; + forge_core::refs::read_merge_base( + self.client, + &core, + &repo.scope()?, + base_ref, + opened_at, + freshness, + ) + .await + } + + fn refs_changed(&self) { + self.collab.refs_changed(); + } + + async fn balance(&self, identity: &str) -> Option { + self.client.get_balance(identity).await.ok() + } + + async fn create_imported( + &self, + repo: &RepoRef, + what: ImportedTarget<'_>, + from: Provenance<'_>, + ) -> Result { + self.collab.create_imported(repo, what, from).await + } + + async fn comment( + &self, + repo: &RepoRef, + target_id: &str, + body: &str, + anchor: Option<&CommentAnchor>, + imported: &Imported, + ) -> Result { + self.collab + .comment(repo, target_id, body, anchor, Some(imported)) + .await + } + + async fn review( + &self, + repo: &RepoRef, + patch_id: &str, + commit_oid: &[u8], + body: &str, + imported: &Imported, + ) -> Result { + self.collab + .review( + repo, + patch_id, + Verdict::Comment, + commit_oid, + body, + None, + Some(imported), + ) + .await + } + + async fn post_event( + &self, + repo: &RepoRef, + target: &Target, + kind: EventKind, + value: &str, + ) -> Result { + self.collab + .post_event(repo, target, kind, Some(value), None) + .await + } + + async fn write_transition( + &self, + repo: &RepoRef, + target: &Target, + mv: &TransitionMove, + merge_oid: Option<&[u8]>, + ) -> Result { + self.collab + .write_transition(repo, target, mv, merge_oid) + .await + } + + async fn create_label( + &self, + repo: &RepoRef, + name: &str, + color: &str, + description: &str, + ) -> Result { + self.collab + .create_label(repo, name, color, description, false) + .await + } + + async fn create_release(&self, repo: &RepoRef, input: &ReleaseInput) -> Result { + self.collab.create_release(repo, input).await + } + + async fn sync_sealed_releases( + &self, + ledger: &mut Ledger<'_>, + repo: &RepoRef, + releases: &[SrcRelease], + storage: Option<&ReleaseStorage>, + ) -> anyhow::Result<()> { + use anyhow::Context as _; + let targets = storage + .map(ReleaseStorage::targets) + .transpose() + .context("opening the release storage")?; + let dest = CollabDest { + collab: &self.collab, + repo, + store: targets.as_ref().map(ReleaseTargets::store), + }; + let fetch = crate::assets::Https::default(); + crate::sealed_release::sync(ledger, &dest, releases, &fetch).await + } +} diff --git a/crates/forge-import/src/dest.rs b/crates/forge-import/src/dest.rs index e15367f15..667065cab 100644 --- a/crates/forge-import/src/dest.rs +++ b/crates/forge-import/src/dest.rs @@ -18,6 +18,7 @@ use forge_core::collab::v2::Collab; use forge_core::repo::credits_to_dash; use crate::budget::{Budget, CapExceeded}; +use crate::chain::CollabChain; use crate::gitsync::ProofRepo; use crate::model::SrcCollab; use crate::sealed_release::ReleaseStorage; @@ -344,21 +345,23 @@ pub async fn dry_collab<'a>( src: &SrcCollab, mirror: Option, release_storage: Option, + lanes: usize, ) -> Result> { // A private destination is read with the signer's keys (its documents are sealed). let collab = match signer { Some(s) => Collab::new(client, &s.identity, &s.bridge), None => Collab::reader(client), }; - let mut dry = Sink::new( - collab, + let dry = Sink::new( + CollabChain::new(client, collab), existing, Ledger::new(client, signer.map(Signer::id), true, Budget::new(None)), ) .with_mirror(mirror) - .with_release_storage(release_storage); + .with_release_storage(release_storage) + .with_lanes(lanes); dry.sync(src).await?; - Ok(dry.ledger) + Ok(dry.into_ledger()) } /// What of `src` is written to a destination the mirror identity holds `role` in, and the @@ -432,6 +435,8 @@ pub struct CollabSource<'s> { pub mirror: Option, /// Where a private destination's sealed release files and asset lists go. pub release_storage: Option, + /// Lanes for the items' dependent writes ([`crate::pipeline`]; 1: one write at a time). + pub lanes: usize, } /// Write the collaboration documents missing from `repo` with the run's ledger (taken from @@ -450,6 +455,7 @@ pub async fn write_collab<'a>( src, mirror, release_storage, + lanes, } = source; let mut ledger = outcome.ledger.take().expect("the write phase has a ledger"); let private = repo.visibility == Visibility::Private; @@ -457,15 +463,19 @@ pub async fn write_collab<'a>( for w in warnings { ledger.warn(w); } - let mut sink = Sink::new( - Collab::new(client, &signer.identity, &signer.bridge), + let sink = Sink::new( + CollabChain::new( + client, + Collab::new(client, &signer.identity, &signer.bridge), + ), Some(repo), ledger, ) .with_mirror(mirror) - .with_release_storage(release_storage); + .with_release_storage(release_storage) + .with_lanes(lanes); let result = sink.sync(&plan).await; - outcome.ledger = Some(sink.ledger); + outcome.ledger = Some(sink.into_ledger()); result } diff --git a/crates/forge-import/src/importer.rs b/crates/forge-import/src/importer.rs index 0412a4ee8..05aaefa7c 100644 --- a/crates/forge-import/src/importer.rs +++ b/crates/forge-import/src/importer.rs @@ -53,6 +53,9 @@ pub struct ImportConfig { pub network: NetworkTarget, /// The signing identity source; optional for a dry run. pub key: Option, + /// Lanes for each issue's and PR's dependent writes (comments, reviews, labels, state), + /// behind its create ([`crate::pipeline`]); 1 writes one document at a time. + pub concurrency: usize, } /// Run an import. Always returns a summary (a failure is its status and error; what was @@ -255,6 +258,7 @@ async fn run_inner<'a>( &priced, mirror.clone(), release_storage.clone(), + cfg.concurrency, ) .await?; let collab_estimate = dry.budget.spent(); @@ -362,6 +366,7 @@ async fn run_inner<'a>( src: &collab_src, mirror, release_storage, + lanes: cfg.concurrency, }; dest::write_collab(client, signer, role, repo, source, definitions, outcome).await?; diff --git a/crates/forge-import/src/lib.rs b/crates/forge-import/src/lib.rs index 7320fce17..66bb14541 100644 --- a/crates/forge-import/src/lib.rs +++ b/crates/forge-import/src/lib.rs @@ -6,11 +6,15 @@ //! * [`sink`] — diff the desired collaboration state against the chain and write only the //! difference, every write charged to the [`budget`] before it is signed. //! * [`sealed_release`] — releases into a private destination, sealed (private-repos.md §16). +//! * [`pipeline`] — the write phase's schedule: creates in order, each item's other writes in +//! parallel lanes. +//! * [`chain`] — what the sink reads from and writes to the destination. //! * [`gitsync`] — git data through the ordinary `git-remote-dash` push. //! * [`summary`] — the run summary (table and `--summary-json`). pub mod assets; pub mod budget; +pub mod chain; pub mod claim; pub mod dest; pub mod github; @@ -18,6 +22,7 @@ pub mod gitlab; pub mod gitsync; pub mod importer; pub mod model; +pub mod pipeline; pub mod sealed_release; pub mod sink; pub mod source; @@ -25,3 +30,6 @@ pub mod source_github; pub mod source_gitlab; pub mod state; pub mod summary; + +#[cfg(test)] +mod sink_tests; diff --git a/crates/forge-import/src/main.rs b/crates/forge-import/src/main.rs index bf563e6f8..ad1f98be6 100644 --- a/crates/forge-import/src/main.rs +++ b/crates/forge-import/src/main.rs @@ -82,6 +82,17 @@ struct RunArgs { #[arg(long, default_value_t = 0)] limit: usize, + /// Parallel lanes for each issue's and PR's comments, reviews, labels and state, written + /// behind its create (issues and PRs themselves are always created one at a time, in + /// upstream order). 1 writes one document at a time. + #[arg( + long, + value_name = "N", + default_value_t = forge_import::pipeline::DEFAULT_LANES, + value_parser = forge_import::pipeline::parse_lanes + )] + concurrency: usize, + /// Write the run summary as JSON here (the Mirror Action reads it). #[arg(long)] summary_json: Option, @@ -213,6 +224,7 @@ async fn run(cli: Cli) -> Result { limit: r.limit, network: target(&r.net)?, key: key(&r.net), + concurrency: r.concurrency, }; Ok(finish(&importer::run(&cfg).await, r.summary_json.as_ref())) } @@ -254,6 +266,17 @@ mod tests { assert_eq!(cli.source.as_deref(), Some("o/r")); assert_eq!(cli.run.max_spend, Some(0.05)); assert!(cli.command.is_none()); + assert_eq!(cli.run.concurrency, forge_import::pipeline::DEFAULT_LANES); + } + + #[test] + fn concurrency_is_bounded() { + let parse = |n: &str| Cli::try_parse_from(["forge-import", "o/r", "--concurrency", n]); + assert_eq!(parse("1").unwrap().run.concurrency, 1); + assert_eq!(parse("16").unwrap().run.concurrency, 16); + assert!(parse("0").is_err()); + assert!(parse("17").is_err()); + assert!(parse("x").is_err()); } #[test] diff --git a/crates/forge-import/src/pipeline.rs b/crates/forge-import/src/pipeline.rs new file mode 100644 index 000000000..bf05a020d --- /dev/null +++ b/crates/forge-import/src/pipeline.rs @@ -0,0 +1,475 @@ +//! The pipelined write phase: issue and PR creates strictly in order, each item's dependent +//! writes (comments, reviews, labels, state) in bounded parallel lanes. +//! +//! **Why this split.** Forge numbers issues and PRs densely (forge-v2 §6.2: `number == +//! countOf(issue) + countOf(patch) + 1`), so the creates are one sequence in upstream order, +//! each confirmed before the next ([`run`]'s *head*). Everything else about an item names only +//! the item itself: a comment or review `refersTo` its target (plus `lockGate`, which reads the +//! target's own lock transitions), a label event its target, and a transition's `c1`–`c6` +//! rules read the same target's earlier transitions. None of them moves the count the next +//! create is numbered by. So once item N is confirmed on chain, its *job* (the rest of its +//! writes) can run while N+1… are created. +//! +//! **Order kept.** Each job runs its writes one at a time, in the order the sequential run +//! wrote them: comments (source order), reviews (source order), label events, transitions. +//! Readers order a thread's timeline by `$createdAt`, the state rules need a target's +//! transitions in order, and L-46 puts the state after the thread; one job per item keeps all +//! three. Jobs of different items run in up to `lanes` lanes at once. +//! +//! **Stopping.** The first error that is not about one item (the spend cap, a network error) +//! stops the head from creating more and every lane from starting another write +//! ([`Stop::check`]); writes already in flight finish, so what landed is counted and paid for +//! before the run reports. That error is the run's. +//! +//! `lanes == 1` is the sequential run: create, then that item's writes, then the next item. + +use std::collections::BTreeMap; +use std::future::Future; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Mutex, MutexGuard, PoisonError}; +use std::time::{Duration, Instant}; + +use anyhow::Result; +use futures::channel::mpsc; +use futures::{SinkExt as _, StreamExt as _}; + +/// The default number of lanes: the window git-remote-dash's chunk upload has run at on every +/// network since spike S0.1 (`forge_core::backends::platform::PIPELINE_WINDOW`). +pub const DEFAULT_LANES: usize = forge_core::backends::platform::PIPELINE_WINDOW; + +/// The most lanes a run may use. +/// +/// Every write signs with its own identity-contract nonce (the SDK's nonce cache hands them +/// out under a lock), and Drive accepts them out of order, but only within 24 of the highest +/// that has landed (`MAX_MISSING_IDENTITY_REVISIONS`). A write that stalls (dropped from a +/// mempool, waiting out a quorum rotation) while 24 later ones land can no longer land with +/// its nonce: forge-core's write loop then finds its document absent by id and signs it +/// again with a new nonce (`create_attempts`, up to three times), so a stall costs time, not +/// a document, and never lands one twice. More lanes make that likelier, and the process's DAPI +/// pacing (8 requests a second) caps the useful number near [`DEFAULT_LANES`] anyway. +pub const MAX_LANES: usize = 16; + +/// Parse a lane count (`--concurrency`): 1..=[`MAX_LANES`]. +pub fn parse_lanes(s: &str) -> Result { + match s.parse::() { + Ok(n) if (1..=MAX_LANES).contains(&n) => Ok(n), + _ => Err(format!("expected a number from 1 to {MAX_LANES}")), + } +} + +/// Items the head may place ahead of the lanes, per lane. A crash leaves at most this many +/// items created with their threads unwritten; the next run finds them on chain and finishes +/// them. +const QUEUE_PER_LANE: usize = 4; + +/// Lock `m`, recovering from a panic elsewhere (the state is plain data). +pub(crate) fn lock(m: &Mutex) -> MutexGuard<'_, T> { + m.lock().unwrap_or_else(PoisonError::into_inner) +} + +/// What a lane gets once the run is stopping: it starts no new write. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct Stopped; + +impl std::fmt::Display for Stopped { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("stopping: an earlier write failed") + } +} + +impl std::error::Error for Stopped {} + +/// The stop signal shared by the head and the lanes, and the error that raised it. +#[derive(Debug, Default)] +pub struct Stop { + stopped: AtomicBool, + first: Mutex>, +} + +impl Stop { + /// Whether the run is stopping. + pub fn is_stopped(&self) -> bool { + self.stopped.load(Ordering::SeqCst) + } + + /// Refuse to start another write once the run is stopping. + pub fn check(&self) -> Result<()> { + if self.is_stopped() { + return Err(Stopped.into()); + } + Ok(()) + } + + /// Stop the run with `e`; the first error is the one reported. + pub fn fail(&self, e: anyhow::Error) { + let mut first = lock(&self.first); + if first.is_none() && e.downcast_ref::().is_none() { + *first = Some(e); + } + self.stopped.store(true, Ordering::SeqCst); + } + + fn take(&self) -> Option { + lock(&self.first).take() + } +} + +/// Run `items` through `open` (the head: strictly one after another, in order) and each job it +/// returns through `finish` (up to `lanes` at once). `open` returning `None` means the item has +/// nothing more to do. See the module docs. +pub async fn run( + items: impl IntoIterator, + lanes: usize, + stop: &Stop, + mut open: O, + finish: F, +) -> Result<()> +where + O: FnMut(T) -> OFut, + OFut: Future>>, + F: Fn(J) -> FFut, + FFut: Future>, +{ + if lanes <= 1 { + for item in items { + if let Some(job) = open(item).await? { + finish(job).await?; + } + } + return Ok(()); + } + let (mut tx, rx) = mpsc::channel::(lanes * QUEUE_PER_LANE); + let head = async move { + for item in items { + if stop.is_stopped() { + break; + } + match open(item).await { + Ok(Some(job)) => { + if tx.send(job).await.is_err() { + break; + } + } + Ok(None) => {} + Err(e) => { + stop.fail(e); + break; + } + } + } + // Dropping the sender ends the lanes' stream once they drain the queue. + }; + let finish = &finish; + let tail = rx.for_each_concurrent(lanes, |job| async move { + // Once stopping, queued jobs are dropped unstarted; the next run finds their items + // on chain and finishes them. + if stop.is_stopped() { + return; + } + if let Err(e) = finish(job).await { + stop.fail(e); + } + }); + futures::join!(head, tail); + stop.take().map_or(Ok(()), Err) +} + +/// Progress of the write phase for the log: how many items the head has placed (created, found, +/// or skipped), how many are complete, how many in run order are all complete (and the upstream +/// number of the last of them: with GitHub's one sequence, every item up to it; GitLab's issues +/// and merge requests interleave by creation time), documents written and spend. +#[derive(Debug)] +pub struct Progress { + total: usize, + every: Duration, + state: Mutex, +} + +#[derive(Debug)] +struct ProgressState { + started: Instant, + last: Instant, + printed: bool, + placed: usize, + placed_upstream: u32, + complete: usize, + /// Items at or past `low` that completed, by index, with their upstream numbers. + ahead: BTreeMap, + /// Every item below this index is complete. + low: usize, + low_upstream: u32, +} + +impl Progress { + /// Progress over `total` items, said at most every `every`. + pub fn new(total: usize, every: Duration) -> Self { + let now = Instant::now(); + Self { + total, + every, + state: Mutex::new(ProgressState { + started: now, + last: now, + printed: false, + placed: 0, + placed_upstream: 0, + complete: 0, + ahead: BTreeMap::new(), + low: 0, + low_upstream: 0, + }), + } + } + + /// The head is done with an item (upstream number `upstream`): created it, found it, or + /// skipped it. + pub fn placed(&self, upstream: u32) { + let mut s = lock(&self.state); + s.placed += 1; + s.placed_upstream = s.placed_upstream.max(upstream); + } + + /// Item `index` has nothing more to write this run. + pub fn completed(&self, index: usize, upstream: u32) { + let mut guard = lock(&self.state); + let s = &mut *guard; + s.complete += 1; + s.ahead.insert(index, upstream); + while let Some(n) = s.ahead.remove(&s.low) { + s.low += 1; + s.low_upstream = n; + } + } + + /// The line to log now, when one is due (`last`: the phase is over, said once a line was). + /// `written` counts the documents written so far and `spent_dash` the spend; a dry run + /// (`spent_dash` `None`) counts the documents it would write. + pub fn line(&self, last: bool, written: u64, spent_dash: Option) -> Option { + let mut s = lock(&self.state); + let now = Instant::now(); + let due = if last { + s.printed + } else { + now.duration_since(s.last) >= self.every + }; + if !due { + return None; + } + s.last = now; + s.printed = true; + let elapsed = now.duration_since(s.started).as_secs(); + let (what, docs) = match spent_dash { + Some(dash) => ( + "placed", + format!("{written} documents written, {dash:.6} DASH spent"), + ), + None => ("priced", format!("{written} documents to write (dry run)")), + }; + Some(format!( + "forge-import: progress: {}/{} issues and PRs {what} (up to upstream #{}), {} \ + complete (all of the first {}, through upstream #{}); {docs}; {}m{:02}s", + s.placed, + self.total, + s.placed_upstream, + s.complete, + s.low, + s.low_upstream, + elapsed / 60, + elapsed % 60 + )) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use std::cell::RefCell; + use std::collections::BTreeSet; + use std::rc::Rc; + + /// What the scheduler did, in order: `open i` / `start i` / `end i`. + #[derive(Default)] + struct Trace { + events: Vec, + running: BTreeSet, + most: usize, + opening: bool, + } + + async fn pause(i: usize) { + // Jobs of different lengths, so lanes overtake one another. + tokio::time::sleep(Duration::from_millis(((i * 7) % 5 + 1) as u64 * 10)).await; + } + + #[tokio::test(start_paused = true)] + async fn the_head_is_ordered_and_the_lanes_are_bounded() { + let trace = Rc::new(RefCell::new(Trace::default())); + let stop = Stop::default(); + let (t1, t2) = (trace.clone(), trace.clone()); + run( + 0..40usize, + 4, + &stop, + move |i| { + let t = t1.clone(); + async move { + { + let mut t = t.borrow_mut(); + assert!(!t.opening, "one create at a time"); + t.opening = true; + t.events.push(format!("open {i}")); + } + pause(i + 1).await; + t.borrow_mut().opening = false; + // every third item has nothing more to do + Ok((i % 3 != 0).then_some(i)) + } + }, + move |i| { + let t = t2.clone(); + async move { + { + let mut t = t.borrow_mut(); + assert!( + t.events.contains(&format!("open {i}")), + "a job starts after its item is placed" + ); + assert!(t.running.insert(i), "one job per item"); + t.most = t.most.max(t.running.len()); + t.events.push(format!("start {i}")); + } + pause(i).await; + let mut t = t.borrow_mut(); + t.running.remove(&i); + t.events.push(format!("end {i}")); + Ok(()) + } + }, + ) + .await + .unwrap(); + let t = trace.borrow(); + let opens: Vec = t + .events + .iter() + .filter_map(|e| e.strip_prefix("open ").map(|n| n.parse().unwrap())) + .collect(); + assert_eq!(opens, (0..40).collect::>(), "creates in order"); + assert!( + t.most <= 4 && t.most > 1, + "bounded, and parallel: {}", + t.most + ); + let ends = t.events.iter().filter(|e| e.starts_with("end ")).count(); + assert_eq!(ends, (0..40).filter(|i| i % 3 != 0).count()); + } + + /// The first real error stops the head and every lane from starting more; jobs in flight + /// finish; that error is returned, not a lane's `Stopped`. + #[tokio::test(start_paused = true)] + async fn the_first_error_stops_the_run_and_in_flight_jobs_finish() { + let started = Rc::new(RefCell::new(Vec::new())); + let finished = Rc::new(RefCell::new(Vec::new())); + let stop = Stop::default(); + let (s1, f1) = (started.clone(), finished.clone()); + let stop_ref = &stop; + let err = run( + 0..100usize, + 3, + &stop, + |i| async move { Ok(Some(i)) }, + move |i| { + let (s, f) = (s1.clone(), f1.clone()); + async move { + stop_ref.check()?; + s.borrow_mut().push(i); + if i == 5 { + anyhow::bail!("--max-spend reached"); + } + pause(i).await; + f.borrow_mut().push(i); + Ok(()) + } + }, + ) + .await + .unwrap_err(); + assert_eq!(err.to_string(), "--max-spend reached"); + let started = started.borrow(); + let finished = finished.borrow(); + assert!(started.len() < 20, "stopped early: {started:?}"); + // Every job that started (but #5) finished: nothing was cut off mid-write. + let mut want: Vec = started.iter().copied().filter(|&i| i != 5).collect(); + let mut got = finished.clone(); + want.sort_unstable(); + got.sort_unstable(); + assert_eq!(want, got); + } + + #[tokio::test] + async fn one_lane_is_the_sequential_run() { + let log = RefCell::new(Vec::new()); + run( + 0..5usize, + 1, + &Stop::default(), + |i| { + log.borrow_mut().push(format!("open {i}")); + async move { Ok(Some(i)) } + }, + |i| { + log.borrow_mut().push(format!("finish {i}")); + async { Ok(()) } + }, + ) + .await + .unwrap(); + let want: Vec = (0..5) + .flat_map(|i| [format!("open {i}"), format!("finish {i}")]) + .collect(); + assert_eq!(*log.borrow(), want); + // The head's error is returned as it is. + let e = run( + 0..3usize, + 1, + &Stop::default(), + |i| async move { + anyhow::ensure!(i < 1, "boom"); + Ok(Some(i)) + }, + |_| async { Ok(()) }, + ) + .await + .unwrap_err(); + assert_eq!(e.to_string(), "boom"); + } + + #[test] + fn progress_says_how_far_every_item_is_complete() { + let p = Progress::new(5, Duration::ZERO); + for (i, n) in [10, 11, 13, 14, 15].into_iter().enumerate() { + p.placed(n); + if i != 1 { + p.completed(i, n); + } + } + let line = p.line(false, 42, Some(0.5)).unwrap(); + assert!( + line.contains("5/5 issues and PRs placed (up to upstream #15), 4 complete (all of the first 1, through upstream #10)"), + "{line}" + ); + assert!( + line.contains("42 documents written, 0.500000 DASH spent"), + "{line}" + ); + p.completed(1, 11); + let last = p.line(true, 50, None).unwrap(); + assert!(last.contains("the first 5, through upstream #15"), "{last}"); + assert!(last.contains("5/5 issues and PRs priced"), "{last}"); + assert!(last.contains("50 documents to write (dry run)"), "{last}"); + // A phase that never said anything says nothing at its end either. + let quiet = Progress::new(1, Duration::from_secs(3600)); + assert!(quiet.line(false, 0, Some(0.0)).is_none()); + assert!(quiet.line(true, 0, Some(0.0)).is_none()); + } +} diff --git a/crates/forge-import/src/sink.rs b/crates/forge-import/src/sink.rs index a755717f4..e1ccfcf96 100644 --- a/crates/forge-import/src/sink.rs +++ b/crates/forge-import/src/sink.rs @@ -17,36 +17,42 @@ //! `upstreamNumber`. An earlier item cannot be placed after later ones, so an incremental run //! that finds one missing refuses to start ([`Sink::check_order`]). //! +//! **Pipelined.** The creates stay one strict sequence; each item's other writes (its thread, +//! labels and state) run in up to `lanes` parallel lanes once the item is confirmed +//! ([`crate::pipeline`]). Every lane charges the same budget, and every write is admitted +//! under the ledger's lock ([`Budget::reserve`]). +//! //! A dry run walks the same path with every write replaced by a count and an estimate. use std::collections::{BTreeMap, BTreeSet}; use std::future::Future; use std::path::Path; +use std::sync::{Mutex, MutexGuard}; +use std::time::{Duration, Instant}; use anyhow::{Context, Result}; -use forge_core::collab::v2::{ - Collab, ImportedTarget, PatchInput, PrBase, Provenance, Target, TargetKind, -}; -use forge_core::collab::{ReleaseInput, Verdict}; +use forge_core::collab::v2::{ImportedTarget, PatchInput, PrBase, Provenance, Target, TargetKind}; +use forge_core::collab::ReleaseInput; use forge_core::history::Freshness; use forge_core::platform::PlatformClient; +use forge_core::repo::credits_to_dash; use forge_core::rules::v2::{ - issue_state_v2, next_transition, pr_state_v2, Actor, StateAction, TransitionMove, - TransitionTarget, Visibility, + next_transition, Actor, StateAction, TransitionMove, TransitionTarget, Visibility, }; use forge_core::rules::{EventKind, MergeBaseTips}; use forge_core::scope::RepoRef; use crate::budget::{collab_doc_credits, Budget, CollabDoc}; +use crate::chain::{Chain, Current}; use crate::gitsync::{ProofRepo, Unfetched}; use crate::model::{same_item, same_item_renamed, SrcCollab, SrcLabel, SrcRelease, SrcTarget}; -use crate::sealed_release::{CollabDest, ReleaseStorage, ReleaseTargets}; +use crate::pipeline::{lock, Progress, Stop}; +use crate::sealed_release::ReleaseStorage; use crate::summary::Counts; /// The run's accounting: budget, counts, warnings. pub struct Ledger<'a> { - /// `None` only in the unit tests' offline ledger ([`Ledger::offline`]). client: Option<&'a PlatformClient>, signer: Option, dry_run: bool, @@ -69,43 +75,43 @@ impl<'a> Ledger<'a> { dry_run: bool, budget: Budget, ) -> Self { - Self { - client: Some(client), - signer, - dry_run, - budget, - counts: Counts::default(), - warnings: Vec::new(), - refused: BTreeSet::new(), - } + Self::with_client(Some(client), signer, dry_run, budget) } - /// A ledger with no Platform client, for the unit tests: nothing it does reads the - /// network (no signer, so no balance to reconcile). + /// A ledger that reads no balance itself: its sink measures through its [`Chain`]. + pub fn detached(signer: Option, dry_run: bool, budget: Budget) -> Self { + Self::with_client(None, signer, dry_run, budget) + } + + /// A ledger with no Platform client and no signer, for the unit tests: nothing it does + /// reads the network. #[cfg(test)] pub(crate) fn offline(dry_run: bool) -> Self { + Self::detached(None, dry_run, Budget::new(None)) + } + + /// Whether this is a dry run (every write replaced by a count and an estimate). + pub(crate) fn dry_run(&self) -> bool { + self.dry_run + } + + fn with_client( + client: Option<&'a PlatformClient>, + signer: Option, + dry_run: bool, + budget: Budget, + ) -> Self { Self { - client: None, - signer: None, + client, + signer, dry_run, - budget: Budget::new(None), + budget, counts: Counts::default(), warnings: Vec::new(), refused: BTreeSet::new(), } } - /// The Platform client the destination is read through. - fn client(&self) -> Result<&'a PlatformClient> { - self.client - .context("this ledger is offline: it has no Platform client to read with") - } - - /// Whether this is a dry run (every write replaced by a count and an estimate). - pub(crate) fn dry_run(&self) -> bool { - self.dry_run - } - /// Record a warning (also logged). Redacted: warnings are published (the Action's job /// summary), and some quote URLs or errors from the source. pub fn warn(&mut self, msg: impl Into) { @@ -144,9 +150,10 @@ impl<'a> Ledger<'a> { self.warn(msg); } - /// One write: charged to the budget first (refused past the cap, before anything is - /// signed), counted, executed unless this is a dry run, then reconciled with the - /// measured balance so an estimate that ran low stops the NEXT write. + /// One write outside the pipeline (a sealed release: one at a time, before any issue or + /// PR): charged to the budget first (refused past the cap, before anything is signed), + /// counted, executed unless this is a dry run, then reconciled with the measured balance so + /// an estimate that ran low stops the NEXT write. pub async fn write( &mut self, what: String, @@ -173,41 +180,23 @@ impl<'a> Ledger<'a> { /// The signer's balance, read only when the per-write cost trace is on /// (`RUST_LOG=forge_import::cost=debug`): it costs a query per write. pub async fn traced_balance(&self) -> Option { - if !tracing::enabled!(target: "forge_import::cost", tracing::Level::DEBUG) { + if !cost_traced() { return None; } self.balance(self.signer.as_deref()?).await } - /// Log one write's estimate against its measured balance drop (calibration). Nodes can - /// answer from a height before the write landed, so the balance is re-read (up to ~10 s) - /// until it moves; every write costs something. + /// Log one write's estimate against its measured balance drop (calibration). pub async fn trace_cost(&self, what: &str, estimated: u64, before: Option) { - let Some(before) = before else { return }; - let mut after = self.traced_balance().await; - for _ in 0..10 { - if after.is_some_and(|a| a != before) { - break; - } - tokio::time::sleep(std::time::Duration::from_secs(1)).await; - after = self.traced_balance().await; - } - if let Some(after) = after { - tracing::debug!( - target: "forge_import::cost", - what, - estimated, - measured = before.saturating_sub(after), - "write cost" - ); - } + trace_cost(what, estimated, before, || self.traced_balance()).await; } /// Pull the measured balance drop into the budget; whether the balance could be read. pub async fn reconcile(&mut self) -> bool { - if let (Some(signer), Some(client)) = (&self.signer, self.client) { + let mark = self.budget.mark(); + if let (Some(client), Some(signer)) = (self.client, &self.signer) { if let Ok(balance) = client.get_balance(signer).await { - self.budget.reconcile(balance); + self.budget.measured(balance, mark); return true; } } @@ -218,28 +207,60 @@ impl<'a> Ledger<'a> { pub async fn balance(&self, identity: &str) -> Option { self.client?.get_balance(identity).await.ok() } +} - fn is_mine(&self, author: &str) -> bool { - self.signer.as_deref().is_none_or(|s| s == author) +/// Whether the per-write cost trace is on (`RUST_LOG=forge_import::cost=debug`). It reads the +/// balance around each write, which writes in flight together would blur: a traced run +/// writes one at a time. +pub fn cost_traced() -> bool { + tracing::enabled!(target: "forge_import::cost", tracing::Level::DEBUG) +} + +/// Log one write's estimate against its measured balance drop (calibration). Nodes can answer +/// from a height before the write landed, so the balance is re-read (up to ~10 s, `read`) +/// until it moves; every write costs something. +async fn trace_cost(what: &str, estimated: u64, before: Option, read: F) +where + F: Fn() -> Fut, + Fut: Future>, +{ + let Some(before) = before else { return }; + let mut after = read().await; + for _ in 0..10 { + if after.is_some_and(|a| a != before) { + break; + } + tokio::time::sleep(Duration::from_secs(1)).await; + after = read().await; + } + if let Some(after) = after { + tracing::debug!( + target: "forge_import::cost", + what, + estimated, + measured = before.saturating_sub(after), + "write cost" + ); } } -/// The destination and its accounting. -pub struct Sink<'a> { - collab: Collab<'a>, - /// `None` only in a dry run whose destination does not exist yet. - repo: Option, - /// Budget, counts, warnings. - pub ledger: Ledger<'a>, +/// How often lanes in flight together read the balance for the spend guard: at most once in +/// this long across all of them (each read is a DAPI request). Their reservations cover the +/// writes that finished since ([`Budget::spent`]). One lane reads after every write. +const MEASURE_EVERY: Duration = Duration::from_secs(2); + +/// How often the write phase logs its progress. +const PROGRESS_EVERY: Duration = Duration::from_secs(30); + +/// What the sink caches while it runs (read once, or kept from what it wrote). +#[derive(Default)] +struct Caches { /// Every imported issue and PR in the destination, `(imported.url, target)`, whoever /// wrote it; loaded on the first miss of a point lookup. index: Option>, /// The destination's members (maintainers and writers), read once: an imported item by /// a member is an earlier mirror's copy; one by anyone else is a squatter. members: Option>, - /// The git data a merge commit's ancestry is checked in, when this run has any (see - /// [`Sink::merge_proof`]). - mirror: Option, /// Each destination PR's base as the readers fold it, by target `$id`. Filled wherever /// a patch is read. opened: BTreeMap, @@ -249,9 +270,10 @@ pub struct Sink<'a> { /// PRs (destination `$id`) whose base was not a branch on chain when they were mirrored: /// no merge into it ever counts (D-501), so a later run cannot prove them either. never_provable: BTreeSet, - /// Bases this write phase read from the network already. The process's synced copy of - /// the history may be the dry run's, from before this run's push: a base's first read - /// here asks the network (`Freshness::Now`), later ones reuse that copy. + /// Bases this write phase has read from the network. The process's synced copy of the + /// history may be the dry run's, from before this run's push: until a base's first read + /// here has come back, reads ask the network (`Freshness::Now`), later ones reuse that + /// copy. fresh_read: BTreeSet, /// Each source item looked up this run, by `(tk, source number)`: its copy, or `None` /// when it has none (yet). @@ -259,6 +281,39 @@ pub struct Sink<'a> { /// The first source item stored at a number other than its own ([`Sink::create`]): /// numbers diverge from there on, said once. diverged: bool, +} + +/// An item placed on chain (created, or found) whose other writes are still to do: a lane's +/// job ([`crate::pipeline`]). +struct Job<'s> { + index: usize, + t: &'s SrcTarget, + target: Target, + fresh: bool, +} + +/// The destination and its accounting. Its methods take `&self`: the pipelined write phase +/// runs several at once, over state behind locks that are never held across an await. +pub struct Sink<'a, C: Chain> { + chain: C, + /// `None` only in a dry run whose destination does not exist yet. + repo: Option, + /// Budget, counts, warnings. + ledger: Mutex>, + /// The signer (copied from the ledger: read on every item). + signer: Option, + dry_run: bool, + /// The git data a merge commit's ancestry is checked in, when this run has any (see + /// [`Sink::merge_proof`]). + mirror: Option, + caches: Mutex, + /// Lanes for the items' dependent writes (1: one write at a time). + lanes: usize, + stop: Stop, + /// When lanes last read the balance ([`MEASURE_EVERY`]). + measured_at: Mutex>, + /// Held by the lane waiting out a base's read-after-write lag ([`Sink::merge_proof`]). + lag_gate: futures::lock::Mutex<()>, /// Where a private destination's sealed release files and asset lists go (`None`: the /// storage policy names no storage of your own). release_storage: Option, @@ -330,16 +385,9 @@ fn state_after(thread: &Result<()>) -> bool { thread.as_ref().err().is_none_or(item_error) } -/// A target's state on chain: its state code (the sum of its transitions) and its labels. -#[derive(Debug, Clone, Default, PartialEq, Eq)] -struct Current { - code: i64, - labels: BTreeSet, -} - /// The state code `t` has at the source: an issue open (0) or closed (1); a PR open (0), /// closed (1), merged (2), and a draft adds 8 unless merged. -fn wanted_code(t: &SrcTarget) -> i64 { +pub(crate) fn wanted_code(t: &SrcTarget) -> i64 { let closed = i64::from(t.closed || t.merged_oid.is_some()); match t.kind { TargetKind::Issue => closed, @@ -352,7 +400,12 @@ fn wanted_code(t: &SrcTarget) -> i64 { /// `to` (breadth first over the legal moves, [`next_transition`]); empty when it already is /// there, or cannot get there (a merged PR is terminal). A merge is only a step when `to` is /// merged. -fn state_path(target: TransitionTarget, from: i64, to: i64, number: u32) -> Vec { +pub(crate) fn state_path( + target: TransitionTarget, + from: i64, + to: i64, + number: u32, +) -> Vec { use std::collections::VecDeque; const ACTIONS: [StateAction; 5] = [ StateAction::Close, @@ -475,22 +528,22 @@ fn text_doc(text: &str) -> u64 { text.len() as u64 + 160 } -impl<'a> Sink<'a> { - /// A sink for `repo` (see [`Ledger`]). - pub fn new(collab: Collab<'a>, repo: Option, ledger: Ledger<'a>) -> Self { +impl<'a, C: Chain> Sink<'a, C> { + /// A sink for `repo` (see [`Ledger`]), writing one item at a time until + /// [`Self::with_lanes`]. + pub fn new(chain: C, repo: Option, ledger: Ledger<'a>) -> Self { Self { - collab, + chain, repo, - ledger, - index: None, - members: None, + signer: ledger.signer.clone(), + dry_run: ledger.dry_run, + ledger: Mutex::new(ledger), mirror: None, - opened: BTreeMap::new(), - lag_waited: BTreeSet::new(), - never_provable: BTreeSet::new(), - fresh_read: BTreeSet::new(), - known: BTreeMap::new(), - diverged: false, + caches: Mutex::new(Caches::default()), + lanes: 1, + stop: Stop::default(), + measured_at: Mutex::new(None), + lag_gate: futures::lock::Mutex::new(()), release_storage: None, } } @@ -509,12 +562,112 @@ impl<'a> Sink<'a> { self } + /// Run the items' dependent writes in up to `lanes` lanes (1..=[`crate::pipeline::MAX_LANES`]; + /// 1 when a per-write cost trace is on, this crate's or forge-core's + /// `DASH_FORGE_COST_TRACE=1`, which read the balance around each write). + #[must_use] + pub fn with_lanes(mut self, lanes: usize) -> Self { + self.lanes = if cost_traced() || forge_core::backends::platform::pipeline_window() == 1 { + 1 + } else { + lanes.clamp(1, crate::pipeline::MAX_LANES) + }; + self + } + + /// The ledger, once the sink is done. + pub fn into_ledger(self) -> Ledger<'a> { + self.ledger + .into_inner() + .unwrap_or_else(std::sync::PoisonError::into_inner) + } + + fn ledger(&self) -> MutexGuard<'_, Ledger<'a>> { + lock(&self.ledger) + } + + fn caches(&self) -> MutexGuard<'_, Caches> { + lock(&self.caches) + } + + fn is_mine(&self, author: &str) -> bool { + self.signer.as_deref().is_none_or(|s| s == author) + } + + fn warn(&self, msg: impl Into) { + self.ledger().warn(msg); + } + + /// One write: charged to the budget first (refused past the cap, before anything is + /// signed; reserved while other lanes write), counted, executed unless this is a dry run, + /// then reconciled with the measured balance so an estimate that ran low stops a later + /// write. Refused once the run is stopping ([`Stop`]). + async fn write( + &self, + what: String, + credits: u64, + count: fn(&mut Counts), + f: F, + ) -> Result> + where + F: FnOnce() -> Fut, + Fut: Future>, + { + self.stop.check()?; + { + let mut ledger = self.ledger(); + if self.dry_run { + ledger.budget.charge(credits, what.clone())?; + } else { + ledger.budget.reserve(credits, what.clone())?; + } + count(&mut ledger.counts); + } + if self.dry_run { + return Ok(None); + } + let before = self.traced_balance().await; + let out = f().await; + self.ledger().budget.finish(credits); + self.measure(false).await; + let out = out.with_context(|| format!("writing {what}"))?; + trace_cost(&what, credits, before, || self.traced_balance()).await; + Ok(Some(out)) + } + + /// The signer's balance, when the per-write cost trace is on (lanes are 1 then). + async fn traced_balance(&self) -> Option { + if !cost_traced() { + return None; + } + self.chain.balance(self.signer.as_deref()?).await + } + + /// Pull the measured balance drop into the budget: after every write with one lane; with + /// more, at most every [`MEASURE_EVERY`] across them, unless `now`. + async fn measure(&self, now: bool) { + let Some(signer) = self.signer.as_deref() else { + return; + }; + if self.lanes > 1 && !now { + let mut at = lock(&self.measured_at); + if at.is_some_and(|t| t.elapsed() < MEASURE_EVERY) { + return; + } + *at = Some(Instant::now()); + } + let mark = self.ledger().budget.mark(); + if let Some(balance) = self.chain.balance(signer).await { + self.ledger().budget.measured(balance, mark); + } + } + /// The base tips a reader checks PR `target_id`'s merge against: the base it was opened /// against, at its `$createdAt` ([`pr_base_tips`], as `Collab::patch_view` folds it). A PR /// this run has not read was opened now (after this run's push). None without a /// destination (a dry run of a repo not created yet). async fn base_tips( - &mut self, + &self, target_id: &str, base_ref: &str, freshness: Freshness, @@ -522,40 +675,36 @@ impl<'a> Sink<'a> { let PrBase { ref_name: base_ref, opened_at, - } = self.opened.get(target_id).cloned().unwrap_or(PrBase { - ref_name: base_ref.to_string(), - opened_at: u64::MAX, - }); + } = self + .caches() + .opened + .get(target_id) + .cloned() + .unwrap_or(PrBase { + ref_name: base_ref.to_string(), + opened_at: u64::MAX, + }); let Some(repo) = self.repo.as_ref() else { return Ok(MergeBaseTips::default()); }; - // A private repo's ref names are sealed: Collab reads (and caches) its updates. - if repo.visibility == Visibility::Private { - return Ok(self - .collab - .base_ref_tips(repo, &base_ref, opened_at) - .await?); - } // One read of the repo's git history per process (`Synced`), shared by every PR of // the run; `Now` when this run's own push may not show yet, and for a base's first - // read in a write phase (the synced copy may predate this run's push). - let first_write_read = !self.ledger.dry_run && self.fresh_read.insert(base_ref.clone()); + // read in a write phase (the synced copy may predate this run's push). Lanes that ask + // before that first read has come back ask the network too. + let first_write_read = !self.dry_run && !self.caches().fresh_read.contains(&base_ref); let freshness = if first_write_read { Freshness::Now } else { freshness }; - let client = self.ledger.client()?; - let core = client.fetch_contract(&repo.forge().core).await?; - Ok(forge_core::refs::read_merge_base( - client, - &core, - &repo.scope()?, - &base_ref, - opened_at, - freshness, - ) - .await?) + let tips = self + .chain + .merge_base(repo, &base_ref, opened_at, freshness) + .await?; + if first_write_read { + self.caches().fresh_read.insert(base_ref); + } + Ok(tips) } /// The oid a merge event for `t` names so that every reader counts it (D-602), or `None` @@ -576,7 +725,7 @@ impl<'a> Sink<'a> { /// /// A run without `code` pushes nothing: the git data is the base branches fetched for the /// proof ([`crate::gitsync::fetch_proof_bases`]), and only tips already on chain count. - async fn merge_proof(&mut self, t: &SrcTarget, target_id: &str) -> Result>> { + async fn merge_proof(&self, t: &SrcTarget, target_id: &str) -> Result>> { let (Some(merged), Some(patch), Some(proof)) = (&t.merged_oid, &t.patch, self.mirror.as_ref()) else { @@ -591,17 +740,19 @@ impl<'a> Sink<'a> { let local = pushed_tip(proof, base, &merged); let (dir, pushed) = (proof.dir.clone(), proof.pushed); let contains = |tips: &MergeBaseTips| chain_tip_containing(&dir, tips, &merged); + let proven = |tips: &MergeBaseTips| contains(tips).and_then(|tip| hex::decode(tip).ok()); let mut tips = self.base_tips(target_id, base, Freshness::Synced).await?; + let opened = self.caches().opened.get(target_id).cloned(); // A PR opened against a base that did not exist then has no tips, and no merge into // it ever counts (D-501); naming one would be re-posted, and paid for, every run. - let base_counts = tips.tip.is_some() || !self.opened.contains_key(target_id); + let base_counts = tips.tip.is_some() || opened.is_none(); // Never provable: the PR was opened against a base with no tip on chain (read from // chain), or this run pushes nothing and the base never had a tip, so the PR it just // created is opened against a base that is no branch (D-501). if !base_counts || (!pushed && tips.historical.is_empty()) { - self.never_provable.insert(target_id.to_string()); + self.caches().never_provable.insert(target_id.to_string()); } - if self.ledger.dry_run { + if self.dry_run { return Ok(contains(&tips) .or(local.filter(|_| base_counts)) .and_then(|tip| hex::decode(tip).ok())); @@ -612,55 +763,65 @@ impl<'a> Sink<'a> { // Only where the wait can help: a base readers count for this PR, and the same ref // the destination PR folds against (a PR retargeted at the source keeps its original // base on chain); and once per base per run. - let folded = self - .opened - .get(target_id) + let settled = |tips: &MergeBaseTips| { + contains(tips).is_some() || local.as_ref().is_none_or(|l| tips.contains(l)) + }; + if settled(&tips) { + return Ok(proven(&tips)); + } + // One lane waits at a time. A lane that finds another waiting reads, once that one is + // done, the copy it brought up to date (`Synced`), as the next PR of a sequential run + // would, instead of settling for what it read before the wait. + let _gate = self.lag_gate.lock().await; + tips = self.base_tips(target_id, base, Freshness::Synced).await?; + let folded = opened + .as_ref() .map_or(base.as_str(), |b| b.ref_name.as_str()); - let may_lag = base_counts && folded == base && !self.lag_waited.contains(base); + let may_lag = + base_counts && folded == base && !self.caches().lag_waited.contains(base.as_str()); let waits = if may_lag { BASE_LAG_WAITS } else { &[] }; for wait in waits { - let seen = local.as_ref().is_none_or(|l| tips.contains(l)); - if contains(&tips).is_some() || seen { + if settled(&tips) { break; } // Marked only when a wait happens: a PR whose merge the mirror lacks (no `local`) // must not use up the one wait a later PR on the same base may need. - self.lag_waited.insert(base.clone()); - tokio::time::sleep(std::time::Duration::from_millis(*wait)).await; - self.collab.refs_changed(); + self.caches().lag_waited.insert(base.clone()); + tokio::time::sleep(Duration::from_millis(*wait)).await; + self.chain.refs_changed(); tips = self.base_tips(target_id, base, Freshness::Now).await?; } - Ok(contains(&tips).and_then(|tip| hex::decode(tip).ok())) + Ok(proven(&tips)) } /// Whether `author` may have mirrored an item: the signer, or a member of the /// destination. Issues and PRs are ungated, so anyone else's `imported` is a claim, not a /// copy. - async fn trusted(&mut self, author: &str) -> Result { - if self.ledger.is_mine(author) { + async fn trusted(&self, author: &str) -> Result { + if self.is_mine(author) { return Ok(true); } - if self.members.is_none() { - let members = match &self.repo { - Some(repo) => forge_core::members::MemberReader::new(self.ledger.client()?) - .list(repo) - .await - .context("reading the destination's members")? - .into_iter() - .map(|m| m.identity_id) - .collect(), - None => BTreeSet::new(), - }; - self.members = Some(members); + if let Some(members) = &self.caches().members { + return Ok(members.contains(author)); } - Ok(self.members.as_ref().is_some_and(|m| m.contains(author))) + let members = match &self.repo { + Some(repo) => self + .chain + .members(repo) + .await + .context("reading the destination's members")?, + None => BTreeSet::new(), + }; + let trusted = members.contains(author); + self.caches().members = Some(members); + Ok(trusted) } /// Whether the document `target` (key `url`) is the mirrored copy of `t`: the same /// source item, written by the signer (a renamed source repo still matches) or by a /// member of the destination (exact repository only). - async fn is_copy(&mut self, url: &str, target: &Target, t: &SrcTarget) -> Result { - let mine = self.ledger.is_mine(&target.author); + async fn is_copy(&self, url: &str, target: &Target, t: &SrcTarget) -> Result { + let mine = self.is_mine(&target.author); // Membership is read only when it can change the answer. let member = !mine && target.kind == t.kind @@ -676,7 +837,7 @@ impl<'a> Sink<'a> { } /// Mirror everything in `src`: label definitions, releases, then issues and PRs. - pub async fn sync(&mut self, src: &SrcCollab) -> Result<()> { + pub async fn sync(&self, src: &SrcCollab) -> Result<()> { if let Some(labels) = &src.labels { self.sync_labels(labels).await?; } @@ -689,16 +850,61 @@ impl<'a> Sink<'a> { } } self.check_order(src).await?; - for t in &src.targets { - self.sync_target(t).await?; + self.sync_targets(&src.targets).await + } + + /// Every issue and PR: created (or found) strictly in order, the rest of each item's + /// writes pipelined behind ([`crate::pipeline::run`]). + async fn sync_targets(&self, targets: &[SrcTarget]) -> Result<()> { + let progress = Progress::new(targets.len(), PROGRESS_EVERY); + let progress = &progress; + let result = crate::pipeline::run( + targets.iter().enumerate(), + self.lanes, + &self.stop, + |(index, t)| async move { + let job = self.open_target(index, t).await; + if let Ok(job) = &job { + progress.placed(t.number); + if job.is_none() { + progress.completed(index, t.number); + } + } + self.say(progress, false); + job + }, + |job: Job<'_>| async move { + self.finish_target(&job).await?; + progress.completed(job.index, job.t.number); + self.say(progress, false); + Ok(()) + }, + ) + .await; + if self.lanes > 1 && !self.dry_run { + // What the lanes wrote since their last read. + self.measure(true).await; + } + self.say(progress, true); + result + } + + /// Log the write phase's progress when a line is due. + fn say(&self, progress: &Progress, last: bool) { + let (written, spent) = { + let l = self.ledger(); + (l.counts.item_documents(), l.budget.spent()) + }; + let spent = (!self.dry_run).then(|| credits_to_dash(spent)); + if let Some(line) = progress.line(last, written, spent) { + eprintln!("{line}"); } - Ok(()) } /// The highest source number of `kind` the destination holds from a trusted writer (the /// signer or a member), or 0. A stranger can set any `upstreamNumber`, so theirs are /// skipped; one page usually settles it, else every one is read. - async fn held_upstream(&mut self, kind: TargetKind) -> Result { + async fn held_upstream(&self, kind: TargetKind) -> Result { let Some(repo) = self.repo.clone() else { return Ok(0); }; @@ -706,7 +912,7 @@ impl<'a> Sink<'a> { // every one. Not "a full page": unreadable rows are dropped before this sees them. for all in [false, true] { let rows = self - .collab + .chain .upstream_numbered(&repo, kind, all) .await .context("reading the destination's upstream numbers")?; @@ -739,7 +945,7 @@ impl<'a> Sink<'a> { /// The check is per kind: GitHub numbers issues and PRs in one sequence, but a new issue /// below the highest mirrored PR (or the reverse) is not caught here; it takes the next /// number, and its upstream number stays with it. - async fn check_order(&mut self, src: &SrcCollab) -> Result<()> { + async fn check_order(&self, src: &SrcCollab) -> Result<()> { for kind in [TargetKind::Issue, TargetKind::Patch] { let held = self.held_upstream(kind).await?; let mut below = Vec::new(); @@ -768,7 +974,7 @@ impl<'a> Sink<'a> { missing.len() - 1 ); } - self.ledger.warn(format!( + self.warn(format!( "{} upstream {noun}(s) from #{first} are mirrored after #{held}: Forge \ numbers densely, so they take the next numbers (each keeps its source number \ as its upstream number)", @@ -780,10 +986,10 @@ impl<'a> Sink<'a> { // --- labels and releases ----------------------------------------------------------- - async fn sync_labels(&mut self, labels: &[SrcLabel]) -> Result<()> { + async fn sync_labels(&self, labels: &[SrcLabel]) -> Result<()> { let existing: BTreeMap = match &self.repo { Some(repo) => self - .collab + .chain .labels(repo) .await .context("reading the destination's labels")? @@ -795,7 +1001,7 @@ impl<'a> Sink<'a> { .collect(), None => BTreeMap::new(), }; - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); // Two source labels that clip to the same name would rewrite one definition on every // run; the first wins. let mut seen = BTreeSet::new(); @@ -807,18 +1013,17 @@ impl<'a> Sink<'a> { CollabDoc::Label, (l.name.len() + l.color.len() + l.description.len() + 60) as u64, ); - self.ledger - .write( - format!("label {}", l.name), - credits, - |c| c.labels += 1, - || async move { - collab - .create_label(need(repo)?, &l.name, &l.color, &l.description, false) - .await - }, - ) - .await?; + self.write( + format!("label {}", l.name), + credits, + |c| c.labels += 1, + || async move { + chain + .create_label(need(repo)?, &l.name, &l.color, &l.description) + .await + }, + ) + .await?; } Ok(()) } @@ -838,17 +1043,14 @@ impl<'a> Sink<'a> { /// signatures and common platform builds first ([`crate::model::fit_assets`]), and its /// notes end with a line saying how many are not mirrored, linking the source release /// ([`crate::model::notes_with_footer`]), which readers show. - /// - /// A private destination's releases are sealed instead ([`crate::sealed_release`]). - async fn sync_releases(&mut self, releases: &[SrcRelease]) -> Result<()> { + async fn sync_releases(&self, releases: &[SrcRelease]) -> Result<()> { let mut known = crate::assets::Known::default(); let existing: BTreeMap = match &self.repo { Some(repo) => self - .collab + .chain .releases(repo) .await .context("reading the destination's releases")? - .current .into_iter() .map(|r| { known.add(&r.tag_name, &r.assets); @@ -860,14 +1062,14 @@ impl<'a> Sink<'a> { }; let fetch = crate::assets::Https::default(); let mut budget = crate::assets::RUN_BYTES; - let dry = self.ledger.dry_run; + let dry = self.dry_run; // Newest first (the source's order): the hashing budget goes to the newest releases. let mut prepared = Vec::with_capacity(releases.len()); for r in releases { // A tag the destination would refuse (over 63 bytes, or outside the git ref // grammar the contract enforces, `@{` included) skips that release, not the run. if let Err(e) = forge_core::collab::v2::check_tag_name(&r.tag_name) { - self.ledger.skip(format!("release not mirrored: {e}")); + self.ledger().skip(format!("release not mirrored: {e}")); continue; } let mut assets_in = r.assets.clone(); @@ -881,7 +1083,7 @@ impl<'a> Sink<'a> { ) .await { - self.ledger.warn(w); + self.warn(w); } if dry { // Priced, not fetched: an asset a real run would hash changes the release. @@ -903,10 +1105,13 @@ impl<'a> Sink<'a> { // Counted and warned for the releases this run writes (a re-run that finds them // unchanged says nothing again). let unhashed = assets_in.iter().filter(|a| a.sha256.is_empty()).count(); - self.ledger.counts.assets_omitted += dropped as u64; - self.ledger.counts.assets_unhashed += unhashed as u64; + { + let mut ledger = self.ledger(); + ledger.counts.assets_omitted += dropped as u64; + ledger.counts.assets_unhashed += unhashed as u64; + } if dropped > 0 { - self.ledger.warn(format!( + self.warn(format!( "release {}: {dropped} of its {total} assets do not fit the 4096 bytes a \ release lists; left out (kept first: checksum files, signatures, the \ common platform builds; its notes link the source release)", @@ -925,110 +1130,135 @@ impl<'a> Sink<'a> { assets: assets_in, ..ReleaseInput::default() }; - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); let input = &input; let written = self - .ledger .write( format!("release {}", r.tag_name), credits, |c| c.releases += 1, - || async move { collab.create_release(need(repo)?, input).await }, + || async move { chain.create_release(need(repo)?, input).await }, ) .await; if let Err(e) = written { let item = item_error(&e); - self.ledger.refused_release(&r.tag_name, e, item)?; + self.ledger().refused_release(&r.tag_name, e, item)?; } } Ok(()) } /// A private destination's releases, sealed ([`crate::sealed_release`]), their files and - /// asset lists stored on the run's release storage. - async fn sync_sealed_releases(&mut self, releases: &[SrcRelease]) -> Result<()> { + /// asset lists stored on the run's release storage. They are written before any issue or + /// PR, one at a time, so the ledger is taken out of its lock for them and put back after. + async fn sync_sealed_releases(&self, releases: &[SrcRelease]) -> Result<()> { let Some(repo) = self.repo.as_ref() else { return Ok(()); }; - let targets = self - .release_storage - .as_ref() - .map(ReleaseStorage::targets) - .transpose() - .context("opening the release storage")?; - let dest = CollabDest { - collab: &self.collab, - repo, - store: targets.as_ref().map(ReleaseTargets::store), - }; - let fetch = crate::assets::Https::default(); - crate::sealed_release::sync(&mut self.ledger, &dest, releases, &fetch).await + let placeholder = Ledger::detached(self.signer.clone(), self.dry_run, Budget::new(None)); + let mut ledger = std::mem::replace(&mut *self.ledger(), placeholder); + let result = self + .chain + .sync_sealed_releases(&mut ledger, repo, releases, self.release_storage.as_ref()) + .await; + *self.ledger() = ledger; + result } // --- issues and pull requests -------------------------------------------------------- - /// Mirror one issue or PR. An error that only concerns this item (the destination - /// refused its content) skips it with a warning instead of failing the run, so one bad - /// item cannot stop every future run; spend-cap and network errors still stop the run. - async fn sync_target(&mut self, t: &SrcTarget) -> Result<()> { - match self.sync_target_inner(t).await { - Err(e) if item_error(&e) => { - self.ledger - .skip(format!("{} not mirrored this run: {e:#}", t.imported.url)); - // Only a refusal of the item's own content is final for it; a state move a - // concurrent change beat (a `c*` rule) or a taken number is retried next run. - if content_error(&e) { - self.ledger.refused.insert(key_of(t)); - } - Ok(()) - } - other => other, + /// An error that only concerns item `t` (the destination refused its content) skips it + /// with a warning instead of failing the run, so one bad item cannot stop every future + /// run; spend-cap and network errors still stop the run. + fn item_skipped(&self, t: &SrcTarget, e: anyhow::Error) -> Result<()> { + if !item_error(&e) { + return Err(e); + } + let mut ledger = self.ledger(); + ledger.skip(format!("{} not mirrored this run: {e:#}", t.imported.url)); + // Only a refusal of the item's own content is final for it; a state move a + // concurrent change beat (a `c*` rule) or a taken number is retried next run. + if content_error(&e) { + ledger.refused.insert(key_of(t)); } + Ok(()) } - async fn sync_target_inner(&mut self, t: &SrcTarget) -> Result<()> { + /// The head's part of item `t` (number `index` in the run): find its copy, or create it at + /// the dense next number. `None` when nothing more is written for it. + async fn open_target<'s>(&self, index: usize, t: &'s SrcTarget) -> Result>> { + self.open_target_inner(index, t) + .await + .or_else(|e| self.item_skipped(t, e).map(|()| None)) + } + + async fn open_target_inner<'s>( + &self, + index: usize, + t: &'s SrcTarget, + ) -> Result>> { let noun = t.kind.noun(); let (target, fresh) = if let Some(target) = self.existing(t).await? { (target, false) } else { let target = self.create(t, noun).await?; - if let Some(idx) = &mut self.index { + let mut caches = self.caches(); + if let Some(idx) = &mut caches.index { idx.push((t.imported.url.clone(), target.clone())); } - self.known.insert(key_of(t), Some(target.clone())); + caches.known.insert(key_of(t), Some(target.clone())); (target, true) }; - if !fresh && !self.ledger.is_mine(&target.author) { + if !fresh && !self.is_mine(&target.author) { // A member mirrored this item (an earlier mirror identity, or a second mirror): // writing to it, or recreating it at a new number, would duplicate it. (A // non-member's claim never gets here: it is a squatter, and the item is created // at another number.) Settled: the member's copy is permanent, so it does not hold // the sync state (a mirror identity rotated would otherwise hold it for good). - self.ledger.warn(format!( + self.warn(format!( "{noun} #{} is already mirrored by {} (as #{}); not mirrored again", t.number, target.author, target.number )); - return Ok(()); + return Ok(None); } - let current = if fresh { + Ok(Some(Job { + index, + t, + target, + fresh, + })) + } + + /// A lane's part of an item: its thread, then its state. + async fn finish_target(&self, job: &Job<'_>) -> Result<()> { + self.finish_target_inner(job) + .await + .or_else(|e| self.item_skipped(job.t, e)) + } + + async fn finish_target_inner(&self, job: &Job<'_>) -> Result<()> { + let Job { + t, target, fresh, .. + } = job; + let current = if *fresh { Current::default() } else { - self.current(&target).await? + self.current(target).await? }; // The thread first, then the state: readers order a target's timeline by `$createdAt`, // and at the source a close (or merge) comes after the comments that led to it (L-46). // A comment or review the destination refuses (an item error, which skips this item) // still lets the state be written, as it was before this order: a skip must not also // leave the item open. - let thread = self.sync_thread(t, &target, fresh).await; + let thread = self.sync_thread(t, target, *fresh).await; if state_after(&thread) { - self.sync_state(t, &target, ¤t).await?; + self.sync_state(t, target, ¤t).await?; } thread } /// An item's comments, then (a PR's) reviews. - async fn sync_thread(&mut self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { + async fn sync_thread(&self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { self.sync_comments(t, target, fresh).await?; if t.kind == TargetKind::Patch { self.sync_reviews(t, target, fresh).await?; @@ -1040,28 +1270,30 @@ impl<'a> Sink<'a> { /// each `issue` / `patch` of the repo that carries provenance, whoever wrote it. One /// complete read per kind, so an item is found wherever it landed; read only when its /// upstream number is held by a document that is not its copy. - async fn load_index(&mut self) -> Result<()> { - if self.index.is_some() { + async fn load_index(&self) -> Result<()> { + if self.caches().index.is_some() { return Ok(()); } let mut index = Vec::new(); + let mut opened = Vec::new(); if let Some(repo) = &self.repo { - // through Collab: a private destination's provenance is sealed (§7) let targets = self - .collab + .chain .imported_targets(repo) .await .context("reading the destination's issues and pull requests")?; for row in targets { if let Some(b) = row.base { - self.opened.insert(row.target.id.clone(), b); + opened.push((row.target.id.clone(), b)); } if let Some(i) = row.imported.filter(|i| !i.url.is_empty()) { index.push((i.url, row.target)); } } } - self.index = Some(index); + let mut caches = self.caches(); + caches.opened.extend(opened); + caches.index = Some(index); Ok(()) } @@ -1069,27 +1301,29 @@ impl<'a> Sink<'a> { /// looked up once per run. The `upstream (repoId, upstreamNumber)` index first (one read); /// the full `imported.url` index only when that number is held by something that is not /// its copy (a stranger can write any `upstreamNumber`). - async fn existing(&mut self, t: &SrcTarget) -> Result> { - if let Some(k) = self.known.get(&key_of(t)) { + async fn existing(&self, t: &SrcTarget) -> Result> { + if let Some(k) = self.caches().known.get(&key_of(t)) { return Ok(k.clone()); } let found = self.lookup(t).await?; - self.known.insert(key_of(t), found.clone()); + self.caches().known.insert(key_of(t), found.clone()); Ok(found) } - async fn lookup(&mut self, t: &SrcTarget) -> Result> { - if self.index.is_none() { + async fn lookup(&self, t: &SrcTarget) -> Result> { + if self.caches().index.is_none() { let Some(repo) = self.repo.as_ref() else { return Ok(None); }; - let rows = self.collab.upstream_targets(repo, t.kind, t.number).await?; + let rows = self.chain.upstream_targets(repo, t.kind, t.number).await?; if rows.is_empty() { return Ok(None); } for row in rows { if let Some(b) = &row.base { - self.opened.insert(row.target.id.clone(), b.clone()); + self.caches() + .opened + .insert(row.target.id.clone(), b.clone()); } if let Some(i) = row.imported.filter(|i| !i.url.is_empty()) { if self.is_copy(&i.url, &row.target, t).await? { @@ -1103,8 +1337,9 @@ impl<'a> Sink<'a> { } /// `t` in the full index (when loaded): the first entry that is its copy. - async fn indexed(&mut self, t: &SrcTarget) -> Result> { + async fn indexed(&self, t: &SrcTarget) -> Result> { let candidates: Vec<(String, Target)> = self + .caches() .index .as_ref() .map(|idx| { @@ -1128,18 +1363,18 @@ impl<'a> Sink<'a> { /// again when another create took the number first), with its provenance and its source /// number as `upstreamNumber`. In a dry run the returned target is a placeholder at the /// source number (nothing reads it). - async fn create(&mut self, t: &SrcTarget, noun: &str) -> Result { + async fn create(&self, t: &SrcTarget, noun: &str) -> Result { let placeholder = Target { kind: t.kind, id: String::new(), number: t.number, - author: self.ledger.signer.clone().unwrap_or_default(), + author: self.signer.clone().unwrap_or_default(), }; let credits = collab_doc_credits( CollabDoc::Target, text_doc(&t.title) + t.body.len() as u64 + t.imported.url.len() as u64 + 100, ); - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); let what = format!("{noun} #{}", t.number); let from = Provenance { imported: Some(&t.imported), @@ -1167,17 +1402,18 @@ impl<'a> Sink<'a> { TargetKind::Patch => |c| c.prs += 1, }; let out = self - .ledger .write(what, credits, count, || async move { - collab.create_imported(need(repo)?, doc, from).await + chain.create_imported(need(repo)?, doc, from).await }) .await?; let Some(created) = out else { return Ok(placeholder); }; - if created.number != t.number && !self.diverged { - self.diverged = true; - self.ledger.warn(diverged_note(noun, t, created.number)); + if created.number != t.number { + let first = !std::mem::replace(&mut self.caches().diverged, true); + if first { + self.warn(diverged_note(noun, t, created.number)); + } } Ok(Target { id: created.document_id, @@ -1188,29 +1424,23 @@ impl<'a> Sink<'a> { /// `target`'s state on chain: its state code (the sum of its transitions; "merged" is the /// chain fact, D-9) and its labels as every reader folds them. - async fn current(&mut self, target: &Target) -> Result { + async fn current(&self, target: &Target) -> Result { let repo = need(self.repo.as_ref())?; - let log = self.collab.target_log(repo, &target.id).await?; - let code = log.state_code(); - let labels = match target.kind { - TargetKind::Issue => issue_state_v2(code, &log.events).labels, - TargetKind::Patch => pr_state_v2(code, None, &log.events, None, |_, _| false).labels, - }; - Ok(Current { code, labels }) + Ok(self.chain.current(repo, target).await?) } /// The commit a merge transition for `t` names: a pushed base tip containing the source's /// merge commit when there is one ([`Sink::merge_proof`]), so readers find it on the base; /// else the source's merge commit itself. A merge is recorded either way (D-9: "merged" /// is what a member recorded); readers label one whose commit is not on the base. - async fn merge_oid(&mut self, t: &SrcTarget, target: &Target) -> Result> { + async fn merge_oid(&self, t: &SrcTarget, target: &Target) -> Result> { let upstream = t.merged_oid.clone().unwrap_or_default(); if let Some(tip) = self.merge_proof(t, &target.id).await? { return Ok(tip); } let base = t.patch.as_ref().map_or("?", |p| p.base_ref_name.as_str()); - self.ledger.counts.unproved_merges += 1; - let reason = if self.never_provable.contains(&target.id) { + let never_provable = self.caches().never_provable.contains(&target.id); + let reason = if never_provable { format!( "its base {base} was not a branch on chain when it was mirrored (forge-v2 §6, \ D-501: push the code before the issues and PRs)" @@ -1218,7 +1448,9 @@ impl<'a> Sink<'a> { } else { no_proof_reason(self.mirror.as_ref(), base) }; - self.ledger.warn(format!( + let mut ledger = self.ledger(); + ledger.counts.unproved_merges += 1; + ledger.warn(format!( "{} was merged; recorded as merged with the source's merge commit {}, which readers \ label as not found on the base: {reason}", t.imported.url, @@ -1230,12 +1462,7 @@ impl<'a> Sink<'a> { /// Bring `target` to `t`'s state: label events for the labels that differ, then the /// transitions ([`state_path`]) from its state code to the source's ([`wanted_code`]). /// Idempotent: a target already in that state takes nothing, and a merged one is final. - async fn sync_state( - &mut self, - t: &SrcTarget, - target: &Target, - current: &Current, - ) -> Result<()> { + async fn sync_state(&self, t: &SrcTarget, target: &Target, current: &Current) -> Result<()> { let moves = state_path( t.kind.transition_target(), current.code, @@ -1243,7 +1470,7 @@ impl<'a> Sink<'a> { target.number, ); if let Some(note) = merged_without_sha_note(t, current.code) { - self.ledger.warn(note); + self.warn(note); } let merges = moves .iter() @@ -1253,23 +1480,18 @@ impl<'a> Sink<'a> { } else { None }; - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); for (kind, value) in label_events(t, current) { let credits = collab_doc_credits(CollabDoc::Event, 120 + value.len() as u64); let what = format!("{kind:?} event on #{}", t.number); let value = value.as_str(); - self.ledger - .write( - what, - credits, - |c| c.events += 1, - || async move { - collab - .post_event(need(repo)?, target, kind, Some(value), None) - .await - }, - ) - .await?; + self.write( + what, + credits, + |c| c.events += 1, + || async move { chain.post_event(need(repo)?, target, kind, value).await }, + ) + .await?; } for mv in moves { let oid = (mv.kind == forge_core::rules::transition::PR_MERGE) @@ -1280,35 +1502,37 @@ impl<'a> Sink<'a> { TRANSITION_BYTES + oid.map_or(0, <[u8]>::len) as u64, ); let what = format!("kind-{} transition on #{}", mv.kind, t.number); - self.ledger - .write( - what, - credits, - |c| c.transitions += 1, - || async move { collab.write_transition(need(repo)?, target, &mv, oid).await }, - ) - .await?; + self.write( + what, + credits, + |c| c.transitions += 1, + || async move { chain.write_transition(need(repo)?, target, &mv, oid).await }, + ) + .await?; } Ok(()) } - async fn sync_comments(&mut self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { + /// The source URLs of the documents already on a thread written by the signer. + fn done_by_me(&self, written: Vec) -> BTreeSet { + written + .into_iter() + .filter(|w| self.is_mine(&w.author)) + .filter_map(|w| w.url) + .collect() + } + + async fn sync_comments(&self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { if t.comments.is_empty() { return Ok(()); } - let done: BTreeSet = if fresh { + let done = if fresh { BTreeSet::new() } else { let repo = need(self.repo.as_ref())?; - self.collab - .comments(repo, &target.id) - .await? - .into_iter() - .filter(|c| self.ledger.is_mine(&c.author)) - .filter_map(|c| c.imported.map(|i| i.url)) - .collect() + self.done_by_me(self.chain.comments(repo, &target.id).await?) }; - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); for c in t .comments .iter() @@ -1318,45 +1542,38 @@ impl<'a> Sink<'a> { CollabDoc::Comment, text_doc(&c.body) + c.imported.url.len() as u64, ); - self.ledger - .write( - format!("comment on #{}", t.number), - credits, - |n| n.comments += 1, - || async move { - collab - .comment( - need(repo)?, - &target.id, - &c.body, - c.anchor.as_ref(), - Some(&c.imported), - ) - .await - }, - ) - .await?; + self.write( + format!("comment on #{}", t.number), + credits, + |n| n.comments += 1, + || async move { + chain + .comment( + need(repo)?, + &target.id, + &c.body, + c.anchor.as_ref(), + &c.imported, + ) + .await + }, + ) + .await?; } Ok(()) } - async fn sync_reviews(&mut self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { + async fn sync_reviews(&self, t: &SrcTarget, target: &Target, fresh: bool) -> Result<()> { if t.reviews.is_empty() { return Ok(()); } - let done: BTreeSet = if fresh { + let done = if fresh { BTreeSet::new() } else { let repo = need(self.repo.as_ref())?; - self.collab - .reviews(repo, &target.id) - .await? - .into_iter() - .filter(|r| self.ledger.is_mine(&r.reviewer)) - .filter_map(|r| r.imported.map(|i| i.url)) - .collect() + self.done_by_me(self.chain.reviews(repo, &target.id).await?) }; - let (collab, repo) = (&self.collab, self.repo.as_ref()); + let (chain, repo) = (&self.chain, self.repo.as_ref()); for r in t .reviews .iter() @@ -1366,29 +1583,20 @@ impl<'a> Sink<'a> { CollabDoc::Review, text_doc(&r.body) + r.imported.url.len() as u64 + 40, ); - self.ledger - .write( - format!("review on #{}", t.number), - credits, - |n| n.reviews += 1, - || async move { - collab - .review( - need(repo)?, - &target.id, - // Always a comment: the mirror identity is a member, and a - // member's approve / request-changes counts (§6); a source - // reviewer's verdict must not become one. It is in the body. - Verdict::Comment, - &r.commit_oid, - &r.body, - None, - Some(&r.imported), - ) - .await - }, - ) - .await?; + self.write( + format!("review on #{}", t.number), + credits, + |n| n.reviews += 1, + // Always a comment verdict ([`Chain::review`]): the mirror identity is a + // member, and a member's approve / request-changes counts (§6); a source + // reviewer's verdict must not become one. It is in the body. + || async move { + chain + .review(need(repo)?, &target.id, &r.commit_oid, &r.body, &r.imported) + .await + }, + ) + .await?; } Ok(()) } diff --git a/crates/forge-import/src/sink_tests.rs b/crates/forge-import/src/sink_tests.rs new file mode 100644 index 000000000..0163214ea --- /dev/null +++ b/crates/forge-import/src/sink_tests.rs @@ -0,0 +1,803 @@ +//! The sink against an offline, recorded chain ([`Recorded`]): the documents it writes are +//! kept in memory in landing order, as Platform would hold them, with the rules the pipeline +//! must keep enforced as it writes: dense numbers (`number == count + 1`), a comment, review, +//! event or transition only on an issue or PR that exists (`refersTo`), and a transition only +//! from the state its target is in (`c1`–`c6`). Every write takes a different simulated time, +//! so lanes overtake one another, and a write can "crash the process": it lands, but the +//! client never hears so, and nothing started after it lands. +//! +//! The tests run the real [`Sink`] (diff, resume, pipeline, budget) over it: a pipelined +//! import writes exactly what the sequential one does, each item's documents in the same +//! order; a run crashed mid-pipeline and run again ends with every document once; the spend +//! cap holds with writes in flight together. + +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::Mutex; +use std::time::Duration; + +use forge_core::collab::v2::{ + Created, ImportedRow, ImportedTarget, Provenance, Target, TargetKind, +}; +use forge_core::collab::{CommentAnchor, Imported, Label, Release, ReleaseInput}; +use forge_core::history::Freshness; +use forge_core::network::ForgeIds; +use forge_core::rules::v2::{TransitionMove, Visibility}; +use forge_core::rules::{EventKind, MergeBaseTips}; +use forge_core::scope::RepoRef; + +use crate::budget::Budget; +use crate::chain::{Chain, Current, Written}; +use crate::model::{SrcCollab, SrcComment, SrcPatch, SrcReview, SrcTarget}; +use crate::pipeline::lock; +use crate::sink::{Ledger, Sink}; + +const SIGNER: &str = "mirror"; +const START_BALANCE: u64 = 1_000_000_000_000_000; + +/// One issue or PR on the recorded chain. +#[derive(Debug, Clone)] +struct Item { + target: Target, + url: String, + upstream: u32, + code: i64, + labels: BTreeSet, + /// Its documents, in landing order: `c `, `r `, `e +label`, `t `. + log: Vec, +} + +#[derive(Debug, Default)] +struct State { + items: Vec, + /// Writes begun, in order. + begun: u64, + /// The write that crashes the process: it lands, its caller hears an error, and no write + /// begun after it lands. + crash_at: Option, + crashed: bool, + /// Writes in flight now, and the most at once; creates in flight; per target. + in_flight: usize, + most: usize, + creating: bool, + busy_targets: BTreeSet, + balance: u64, + /// A comment or review on a thread written while another of the same thread was in + /// flight, or a transition refused: must never happen. + violations: Vec, +} + +/// The recorded chain. +struct Recorded { + state: Mutex, + /// What each write really costs, as a share of its estimate (percent). + cost_pct: u64, + /// Source URLs whose document the chain refuses for its content (before it lands). + refuse: BTreeSet, +} + +impl Recorded { + fn new() -> Self { + Self { + state: Mutex::new(State { + balance: START_BALANCE, + ..State::default() + }), + cost_pct: 100, + refuse: BTreeSet::new(), + } + } + + fn st(&self) -> std::sync::MutexGuard<'_, State> { + lock(&self.state) + } + + /// One write on `target_id` (`None`: a create). Waits a simulated time that differs per + /// write, then lands `apply` unless the process crashed before it began. + async fn land( + &self, + target_id: Option<&str>, + apply: impl FnOnce(&mut State) -> forge_core::Result, + ) -> forge_core::Result { + let n = { + let mut st = self.st(); + if st.crashed { + return Err(forge_core::Error::Platform("connection lost".into())); + } + st.begun += 1; + st.in_flight += 1; + st.most = st.most.max(st.in_flight); + match target_id { + None => { + if std::mem::replace(&mut st.creating, true) { + st.violations.push("two creates in flight".into()); + } + } + Some(id) => { + if !st.busy_targets.insert(id.to_string()) { + st.violations.push(format!("two writes in flight on {id}")); + } + } + } + st.begun + }; + // 20–240 ms, different for neighbouring writes. + tokio::time::sleep(Duration::from_millis(20 + (n * 37 % 23) * 10)).await; + let mut st = self.st(); + st.in_flight -= 1; + match target_id { + None => st.creating = false, + Some(id) => { + st.busy_targets.remove(id); + } + } + // Landed: begun before the crash (the crashing write itself included). + let out = apply(&mut st); + if st.crash_at == Some(n) { + st.crashed = true; + } + if st.crashed { + return Err(forge_core::Error::Platform("connection lost".into())); + } + out + } + + fn charge(&self, st: &mut State, estimate: u64) { + st.balance = st.balance.saturating_sub(estimate * self.cost_pct / 100); + } + + fn row(item: &Item) -> ImportedRow { + ImportedRow { + target: item.target.clone(), + imported: Some(Imported { + url: item.url.clone(), + ..Imported::default() + }), + upstream_number: Some(item.upstream), + base: None, + } + } + + fn item<'s>(st: &'s mut State, id: &str) -> &'s mut Item { + st.items + .iter_mut() + .find(|i| i.target.id == id) + .unwrap_or_else(|| panic!("refersTo: no issue or PR {id}")) + } + + /// Every item's log, by upstream number, and the numbers they hold. + fn logs(&self) -> BTreeMap)> { + self.st() + .items + .iter() + .map(|i| (i.upstream, (i.target.number, i.log.clone()))) + .collect() + } +} + +/// What the recorded chain debits for a document (scaled by `cost_pct`): a little under the +/// sink's estimate for its kind (`crate::budget::CollabDoc::index_overhead` alone is 90M for +/// an issue or PR and at least 50M for the rest), as on the real networks. +fn price(kind: &str) -> u64 { + match kind { + "target" => 80_000_000, + _ => 45_000_000, + } +} + +// The reads answer at once; the trait is async for the network. +#[allow(clippy::unused_async_trait_impl)] +impl Chain for &Recorded { + async fn upstream_targets( + &self, + _: &RepoRef, + kind: TargetKind, + n: u32, + ) -> forge_core::Result> { + Ok(self + .st() + .items + .iter() + .filter(|i| i.target.kind == kind && i.upstream == n) + .map(Recorded::row) + .collect()) + } + + async fn upstream_numbered( + &self, + _: &RepoRef, + kind: TargetKind, + all: bool, + ) -> forge_core::Result> { + let mut rows: Vec = self + .st() + .items + .iter() + .filter(|i| i.target.kind == kind) + .map(Recorded::row) + .collect(); + rows.sort_by_key(|r| std::cmp::Reverse(r.upstream_number)); + if !all { + rows.truncate(100); + } + Ok(rows) + } + + async fn imported_targets(&self, _: &RepoRef) -> forge_core::Result> { + Ok(self.st().items.iter().map(Recorded::row).collect()) + } + + async fn current(&self, _: &RepoRef, target: &Target) -> forge_core::Result { + let mut st = self.st(); + let item = Recorded::item(&mut st, &target.id); + Ok(Current { + code: item.code, + labels: item.labels.clone(), + }) + } + + async fn comments(&self, _: &RepoRef, id: &str) -> forge_core::Result> { + let mut st = self.st(); + Ok(written(&Recorded::item(&mut st, id).log, "c ")) + } + + async fn reviews(&self, _: &RepoRef, id: &str) -> forge_core::Result> { + let mut st = self.st(); + Ok(written(&Recorded::item(&mut st, id).log, "r ")) + } + + async fn members(&self, _: &RepoRef) -> forge_core::Result> { + Ok(BTreeSet::new()) + } + + async fn labels(&self, _: &RepoRef) -> forge_core::Result> { + Ok(Vec::new()) + } + + async fn releases(&self, _: &RepoRef) -> forge_core::Result> { + Ok(Vec::new()) + } + + async fn merge_base( + &self, + _: &RepoRef, + _: &str, + _: u64, + _: Freshness, + ) -> forge_core::Result { + Ok(MergeBaseTips::default()) + } + + fn refs_changed(&self) {} + + async fn balance(&self, _: &str) -> Option { + Some(self.st().balance) + } + + async fn create_imported( + &self, + _: &RepoRef, + what: ImportedTarget<'_>, + from: Provenance<'_>, + ) -> forge_core::Result { + let kind = match what { + ImportedTarget::Issue { .. } => TargetKind::Issue, + ImportedTarget::Patch(_) => TargetKind::Patch, + }; + let url = from.imported.map(|i| i.url.clone()).unwrap_or_default(); + let upstream = from.upstream_number.unwrap_or_default(); + if self.refuse.contains(&url) { + return Err(forge_core::Error::RuleRefused { + document_type: kind.doc_type().into(), + rule: "hasTitle".into(), + detail: url, + }); + } + self.land(None, |st| { + // dense: the count of issues and PRs, plus one + let number = u32::try_from(st.items.len()).unwrap() + 1; + let id = format!("doc-{number}"); + st.items.push(Item { + target: Target { + kind, + id: id.clone(), + number, + author: SIGNER.into(), + }, + url, + upstream, + code: 0, + labels: BTreeSet::new(), + log: Vec::new(), + }); + self.charge(st, price("target")); + Ok(Created { + number, + document_id: id, + resumed: false, + draft_transition: None, + }) + }) + .await + } + + async fn comment( + &self, + _: &RepoRef, + target_id: &str, + _: &str, + _: Option<&CommentAnchor>, + imported: &Imported, + ) -> forge_core::Result { + self.append(target_id, format!("c {}", imported.url)).await + } + + async fn review( + &self, + _: &RepoRef, + patch_id: &str, + _: &[u8], + _: &str, + imported: &Imported, + ) -> forge_core::Result { + self.append(patch_id, format!("r {}", imported.url)).await + } + + async fn post_event( + &self, + _: &RepoRef, + target: &Target, + kind: EventKind, + value: &str, + ) -> forge_core::Result { + let add = kind == EventKind::LabelAdd; + let value = value.to_string(); + self.land(Some(&target.id), |st| { + self.charge(st, price("doc")); + let item = Recorded::item(st, &target.id); + if add { + item.labels.insert(value.clone()); + } else { + item.labels.remove(&value); + } + item.log + .push(format!("e {}{value}", if add { '+' } else { '-' })); + Ok(format!("e-{}", item.log.len())) + }) + .await + } + + async fn write_transition( + &self, + _: &RepoRef, + target: &Target, + mv: &TransitionMove, + _: Option<&[u8]>, + ) -> forge_core::Result { + let mv = *mv; + self.land(Some(&target.id), |st| { + // c1–c6: a move from the state the target is in, and nowhere else. + let code = Recorded::item(st, &target.id).code; + if code + mv.delta != mv.after { + let why = format!("kind {} from {code} on {}", mv.kind, target.id); + st.violations.push(format!("refused transition {why}")); + return Err(forge_core::Error::RuleRefused { + document_type: "transition".into(), + rule: "c1_closedAfter".into(), + detail: why, + }); + } + self.charge(st, price("doc")); + let item = Recorded::item(st, &target.id); + item.code = mv.after; + item.log.push(format!("t {}", mv.kind)); + Ok(format!("t-{}", item.log.len())) + }) + .await + } + + async fn create_label( + &self, + _: &RepoRef, + _: &str, + _: &str, + _: &str, + ) -> forge_core::Result { + unreachable!("no label definitions in these tests") + } + + async fn create_release(&self, _: &RepoRef, _: &ReleaseInput) -> forge_core::Result { + unreachable!("no releases in these tests") + } + + async fn sync_sealed_releases( + &self, + _: &mut Ledger<'_>, + _: &RepoRef, + _: &[crate::model::SrcRelease], + _: Option<&crate::sealed_release::ReleaseStorage>, + ) -> anyhow::Result<()> { + unreachable!("no releases in these tests") + } +} + +impl Recorded { + async fn append(&self, target_id: &str, entry: String) -> forge_core::Result { + if self.refuse.contains(&entry[2..]) { + return Err(forge_core::Error::Config("comment body too long".into())); + } + self.land(Some(target_id), |st| { + self.charge(st, price("doc")); + let item = Recorded::item(st, target_id); + item.log.push(entry); + Ok(format!("d-{}", item.log.len())) + }) + .await + } +} + +fn written(log: &[String], prefix: &str) -> Vec { + log.iter() + .filter_map(|e| e.strip_prefix(prefix)) + .map(|url| Written { + author: SIGNER.into(), + url: Some(url.to_string()), + }) + .collect() +} + +fn repo() -> RepoRef { + RepoRef { + forge: ForgeIds::test_forge(), + repo_id: "R".into(), + owner_id: SIGNER.into(), + name: "proj".into(), + visibility: Visibility::Public, + } +} + +fn imported(path: &str) -> Imported { + Imported { + author: "gh-user".into(), + created_at: 1, + url: format!("https://github.com/o/r/{path}"), + } +} + +/// A source of `n` items numbered 1..=n like GitHub's (issues and PRs in one sequence), with +/// threads of different lengths, labels, and every state: open, closed, merged, draft. +fn source(n: u32) -> SrcCollab { + let targets = (1..=n) + .map(|number| { + let pr = number % 3 != 0; + let path = if pr { "pull" } else { "issues" }; + let comments = (0..(number * 7 % 6)) + .map(|k| SrcComment { + body: format!("comment {k}"), + imported: imported(&format!("{path}/{number}#issuecomment-{k}")), + anchor: None, + }) + .collect(); + let reviews = if pr { + (0..(number % 4)) + .map(|k| SrcReview { + verdict: forge_core::collab::Verdict::Comment, + commit_oid: vec![1; 20], + body: format!("review {k}"), + imported: imported(&format!("pull/{number}#pullrequestreview-{k}")), + }) + .collect() + } else { + Vec::new() + }; + let labels = (0..(number % 3)).map(|k| format!("label-{k}")).collect(); + SrcTarget { + kind: if pr { + TargetKind::Patch + } else { + TargetKind::Issue + }, + number, + title: format!("item {number}"), + body: "b".into(), + imported: imported(&format!("{path}/{number}")), + closed: number % 2 == 0, + merged_oid: (pr && number % 4 == 0).then(|| vec![2; 20]), + merged_without_sha: false, + labels, + draft: pr && number % 5 == 1, + patch: pr.then(|| SrcPatch { + base_ref_name: "refs/heads/main".into(), + source_ref_name: None, + head_oid: vec![3; 20], + }), + comments, + reviews, + } + }) + .collect(); + SrcCollab { + targets, + ..SrcCollab::default() + } +} + +/// What each item's log must be after a complete import: its comments and reviews in source +/// order, then its label events, then its transitions (L-46), each once. +fn expected(src: &SrcCollab) -> BTreeMap> { + src.targets + .iter() + .map(|t| { + let mut log: Vec = t + .comments + .iter() + .map(|c| format!("c {}", c.imported.url)) + .collect(); + log.extend(t.reviews.iter().map(|r| format!("r {}", r.imported.url))); + log.extend(t.labels.iter().map(|l| format!("e +{l}"))); + let path = crate::sink::state_path( + t.kind.transition_target(), + 0, + crate::sink::wanted_code(t), + t.number, + ); + log.extend(path.iter().map(|mv| format!("t {}", mv.kind))); + (t.number, log) + }) + .collect() +} + +struct Run { + result: anyhow::Result<()>, + counts: crate::summary::Counts, + spent: u64, + refused: BTreeSet<(u8, u32)>, +} + +fn sink(chain: &Recorded, dry_run: bool, budget: Budget, lanes: usize) -> Sink<'_, &Recorded> { + Sink::new( + chain, + Some(repo()), + Ledger::detached(Some(SIGNER.into()), dry_run, budget), + ) + .with_lanes(lanes) +} + +async fn import(chain: &Recorded, src: &SrcCollab, lanes: usize, cap: Option) -> Run { + let mut budget = Budget::new(cap); + budget.start(chain.st().balance); + let sink = sink(chain, false, budget, lanes); + let result = sink.sync(src).await; + let ledger = sink.into_ledger(); + Run { + result, + counts: ledger.counts, + spent: ledger.budget.spent(), + refused: ledger.refused, + } +} + +async fn dry(chain: &Recorded, src: &SrcCollab, lanes: usize) -> crate::summary::Counts { + let sink = sink(chain, true, Budget::new(None), lanes); + sink.sync(src).await.unwrap(); + sink.into_ledger().counts +} + +fn assert_complete(chain: &Recorded, src: &SrcCollab) { + let want = expected(src); + let got = chain.logs(); + assert_eq!(got.len(), want.len(), "every item once"); + for (upstream, (number, log)) in &got { + // A fresh mirror of a source with no gaps numbers identically, in upstream order. + assert_eq!(number, upstream, "dense numbers in upstream order"); + assert_eq!(log, &want[upstream], "item #{upstream}'s documents"); + } + let st = chain.st(); + assert!(st.violations.is_empty(), "{:?}", st.violations); +} + +/// A pipelined import writes exactly what a sequential one does: every item created in +/// upstream order at its number, each item's documents in the same order, the counts equal to +/// what landed, and the spend equal to the measured drop. And it really runs lanes at once. +#[tokio::test(start_paused = true)] +async fn a_pipelined_import_writes_what_a_sequential_one_does() { + let src = source(40); + let one = Recorded::new(); + let r1 = import(&one, &src, 1, None).await; + r1.result.unwrap(); + assert_complete(&one, &src); + assert!(one.st().most == 1, "one write at a time"); + + let eight = Recorded::new(); + let t0 = tokio::time::Instant::now(); + let r8 = import(&eight, &src, 8, None).await; + let pipelined = t0.elapsed(); + r8.result.unwrap(); + assert_complete(&eight, &src); + assert_eq!(one.logs(), eight.logs()); + let most = eight.st().most; + assert!((3..=9).contains(&most), "lanes plus the head: {most}"); + + // The progress and summary counts are what landed, and the spend what was paid. + assert_eq!(r8.counts, r1.counts); + let landed: u64 = eight.logs().values().map(|(_, l)| l.len() as u64).sum(); + assert_eq!( + r8.counts.item_documents(), + landed + src.targets.len() as u64 + ); + // The spend reported: the estimates (the chain charged a little less), whatever the lanes. + assert_eq!(r8.spent, r1.spent); + assert!(r8.spent >= START_BALANCE - eight.st().balance); + + // Simulated time: the pipeline is several times faster. + let t0 = tokio::time::Instant::now(); + let again = Recorded::new(); + import(&again, &src, 1, None).await.result.unwrap(); + let sequential = t0.elapsed(); + assert!( + pipelined * 3 < sequential, + "pipelined {pipelined:?} vs sequential {sequential:?}" + ); +} + +/// A re-run over a complete mirror writes nothing, sequential or pipelined, and a dry run +/// prices the same at any lane count. +#[tokio::test(start_paused = true)] +async fn a_rerun_writes_nothing_and_dry_runs_agree() { + let src = source(25); + let chain = Recorded::new(); + let fresh_dry = dry(&chain, &src, 1).await; + assert_eq!(dry(&chain, &src, 8).await, fresh_dry); + let first = import(&chain, &src, 8, None).await; + first.result.unwrap(); + assert_eq!( + first.counts, fresh_dry, + "the dry run priced exactly what was written" + ); + let before = chain.logs(); + for lanes in [1, 8] { + let again = import(&chain, &src, lanes, None).await; + again.result.unwrap(); + assert_eq!(again.counts.item_documents(), 0); + assert_eq!(again.spent, 0); + } + assert_eq!(chain.logs(), before); + assert_eq!(dry(&chain, &src, 8).await.item_documents(), 0); +} + +/// The process dies mid-pipeline (a write lands but its caller never hears; nothing begun +/// after it lands). Run again, incrementally (the order check must not refuse it) and +/// pipelined: every item and document ends up on chain exactly once, in order, and the +/// resumed run writes exactly what a dry run between the two said was left. +#[tokio::test(start_paused = true)] +async fn a_run_crashed_mid_pipeline_resumes_to_the_same_mirror() { + let src = source(60); + for crash_at in [1, 7, 60, 131, 190] { + let chain = Recorded::new(); + chain.st().crash_at = Some(crash_at); + let crashed = import(&chain, &src, 8, None).await; + let err = crashed.result.unwrap_err(); + assert!(format!("{err:#}").contains("connection lost"), "{err:#}"); + let partial = chain.logs(); + assert!( + partial.len() < src.targets.len(), + "stopped early at {crash_at}" + ); + // Created items are a prefix in upstream order, at dense numbers. + for (n, (upstream, (number, _))) in (1u32..).zip(partial.iter()) { + assert_eq!((*upstream, *number), (n, n)); + } + { + let mut st = chain.st(); + st.crashed = false; + st.crash_at = None; + } + let left = dry(&chain, &src, 8).await; + let resumed_src = SrcCollab { + incremental: true, + ..src.clone() + }; + let resumed = import(&chain, &resumed_src, 8, None).await; + resumed.result.unwrap(); + assert_eq!( + resumed.counts, left, + "resumed exactly what was left ({crash_at})" + ); + assert_complete(&chain, &src); + } +} + +/// `--max-spend` with lanes in flight together: the run stops at the cap with the spend-cap +/// error, the measured drop never crosses the cap, every write it counted landed (in-flight +/// writes finish before it reports), and a re-run with a higher cap completes the mirror. +#[tokio::test(start_paused = true)] +async fn the_spend_cap_holds_with_lanes_in_flight() { + let src = source(50); + let chain = Recorded::new(); + let cap = 2_500_000_000; + let capped = import(&chain, &src, 8, Some(cap)).await; + let err = capped.result.unwrap_err(); + assert!( + err.downcast_ref::().is_some(), + "{err:#}" + ); + let paid = START_BALANCE - chain.st().balance; + assert!(paid <= cap, "paid {paid} over the cap {cap}"); + assert!(capped.spent <= cap); + let landed: u64 = chain.logs().values().map(|(_, l)| l.len() as u64).sum(); + assert_eq!( + capped.counts.item_documents(), + landed + chain.logs().len() as u64, + "the counts are what landed" + ); + import(&chain, &src, 8, None).await.result.unwrap(); + assert_complete(&chain, &src); +} + +/// Estimates that run low: the measured drop stops the run, pipelined too. What the lanes +/// already had in flight can take it past the cap by at most their own under-estimates. +#[tokio::test(start_paused = true)] +async fn a_low_estimate_is_caught_by_the_measured_drop_with_lanes() { + let src = source(50); + let mut chain = Recorded::new(); + chain.cost_pct = 150; + let cap = 2_500_000_000; + let capped = import(&chain, &src, 8, Some(cap)).await; + assert!(capped.result.is_err()); + let paid = START_BALANCE - chain.st().balance; + // Nine writes in flight at most (eight lanes and the head), each paid at most 1.5 times + // what the chain prices it at (a create: 80M), over its estimate. + let slack = 9 * 120_000_000; + assert!(paid <= cap + slack, "paid {paid}, cap {cap}"); + assert!(capped.spent >= paid, "the ledger reports what was paid"); +} + +/// Items the destination refuses for their content are skipped, in the head (a create) and in +/// a lane (a comment), and the run carries on: the refused create takes no number (the next +/// item takes it), and an item whose comment was refused still gets its labels and state, +/// but nothing after the refused comment. Both are recorded refused, and the result is the +/// same at one lane and at eight. +#[tokio::test(start_paused = true)] +async fn refused_items_are_skipped_and_the_run_carries_on() { + let src = source(30); + let comment = src.targets[9].comments[1].imported.url.clone(); + let create = src.targets[19].imported.url.clone(); + let want = expected(&src); + let mut results = Vec::new(); + for lanes in [1, 8] { + let mut chain = Recorded::new(); + chain.refuse = [comment.clone(), create.clone()].into(); + let run = import(&chain, &src, lanes, None).await; + run.result.unwrap(); + assert_eq!(run.counts.skipped, 2); + let refused: Vec = run.refused.iter().map(|(_, n)| *n).collect(); + assert_eq!(refused, [10, 20]); + let logs = chain.logs(); + assert!( + !logs.contains_key(&20), + "the refused create is not on chain" + ); + for (upstream, (number, log)) in &logs { + let n = if *upstream > 20 { + upstream - 1 + } else { + *upstream + }; + assert_eq!(*number, n, "dense: #{upstream} takes the next number"); + if *upstream != 10 { + assert_eq!(log, &want[upstream], "item #{upstream}"); + } + } + // #10: its first comment, then (past the refused one) its labels and state only. + let mut ten = vec![want[&10][0].clone()]; + ten.extend( + want[&10] + .iter() + .filter(|e| e.starts_with("e ") || e.starts_with("t ")) + .cloned(), + ); + assert_eq!(logs[&10].1, ten); + assert!(chain.st().violations.is_empty()); + results.push(logs); + } + assert_eq!(results[0], results[1]); +} diff --git a/crates/forge-import/src/summary.rs b/crates/forge-import/src/summary.rs index 0fe0a05a3..6d4179e81 100644 --- a/crates/forge-import/src/summary.rs +++ b/crates/forge-import/src/summary.rs @@ -62,6 +62,12 @@ impl Counts { self.packs += push.packs; self.pack_bytes += push.pack_bytes; } + + /// The issues and PRs, and the documents on them (comments, reviews, label events, + /// transitions): what the write phase's progress counts. + pub fn item_documents(&self) -> u64 { + self.issues + self.prs + self.comments + self.reviews + self.events + self.transitions + } } /// How the run ended. diff --git a/docs/guides/mirror-a-github-repo.md b/docs/guides/mirror-a-github-repo.md index df80dea10..135d5f3d1 100644 --- a/docs/guides/mirror-a-github-repo.md +++ b/docs/guides/mirror-a-github-repo.md @@ -101,6 +101,7 @@ Useful flags (see `forge-import --help`): | `--include-label-definitions` | A private destination only: mirror the label definitions too. Their names, colours and descriptions are not encrypted, so they are left out by default; the labels set on issues and PRs are encrypted either way. | | `--limit ` | At most `n` issues and PRs (the oldest), for a cheap trial. Only those items and their comments are read, so a trial on a repository with thousands of issues takes a few requests. | | `--yes` | No confirmation prompt, for CI. | +| `--concurrency ` | How many issues and PRs have their comments, reviews, labels and state written at once (default 8, at most 16). Issues and PRs themselves are always created one at a time, in GitHub's order, so their numbers match. `1` writes one document at a time. See [large repositories](#large-repositories). | | `--summary-json ` | Write the run summary (counts, spend, key budget) as JSON. `forge-import` only. | | `--work-dir ` | Keep the bare git mirror between runs instead of a temporary directory. `forge-import` only. | | `--network`, `--devnet-name`, `--identity` | As for `dg`. `DASH_FORGE_KEY` may be a file path or an inline `dfk1:` key. | @@ -149,6 +150,40 @@ A signature or per-file checksum (`.asc`, `.sig`, `.minisig`, `.sha256`, ` **Not mirrored.** Edits to a title or body after the item was first mirrored, a PR's later retarget to another base, and a PR's later head moves (its `headOid` stays at the commit it was mirrored at; `refs/mirror/pull//head` follows the head while the PR is open). Reactions, milestones, assignees, projects and GitHub Discussions are not mirrored. +### Large repositories + +How the importer writes: +- **Issues and PRs** are created one at a time, in GitHub's order, each confirmed before the next, so their numbers follow GitHub's. +- **Everything else on an item** (its comments, reviews, labels and state) is written behind its creation, while later items are being created. `--concurrency` items (default 8) are written at once, and each item's own documents stay in their order: comments, then reviews, then labels and state. + +**How long it takes.** Timed on devnet bonsia (2026-09-30), importing the issues and PRs of `dashpay/dash-network-deploy` into two new repositories: 749 issues and PRs with 3,911 documents in all (1,097 comments, 1,301 reviews, 764 labels and state changes). + +| | `--concurrency 1` | `--concurrency 8` (the default) | +|---|---|---| +| Writing | 122.6 min (0.53 documents/s) | 38.1 min (1.71 documents/s) | +| Whole run, including about 10 minutes reading GitHub | 132 min | about 48 min | +| Spent (measured balance drop) | 3.164 DASH | 3.138 DASH | + +- **Speed:** writing was 3.2 times as fast; the cost was the same. +- **Interrupted run:** the pipelined run was killed partway through by accident and then run again. The two passes together wrote every document once, and a sample of 47 items (including every item that was mid-write at the kill) matched the other mirror exactly. +- **What limits it:** issues and PRs are still created one at a time, so a repository with many issues and PRs and few comments gains least. +- **dashpay/dash:** about 74,000 documents. It took 10 to 28 hours one document at a time. From these rates, a pipelined import of it should take about 7 to 9 hours. This is an estimate, not a measurement. + +**Progress.** Every 30 seconds the importer prints a progress line to stderr: + +``` +forge-import: progress: 412/749 issues and PRs placed (up to upstream #430), 405 complete (all of the first 398, through upstream #415); 2210 documents written, 1.874210 DASH spent; 9m30s +``` + +- **Complete:** an item whose documents are all written. +- **"all of the first N":** every item up to that point is complete. + +**Interrupted runs.** A run that stops (a crash, a network error, the spend cap) can leave a few dozen items created but not yet complete. Running it again finishes them: it reads each item's comments, reviews and state on chain and writes only what is missing, and it writes nothing twice. + +**Spend cap.** `--max-spend` still holds. Each write reserves its estimate against the cap before it is signed, counting the writes still in flight. When the cap is reached, no further write starts; the writes already in flight finish and are counted in the summary. + +**`--concurrency 1`.** This writes one document at a time, as earlier versions did. Use it when something else signs with the same identity at the same time, or when you measure each write's cost (`RUST_LOG=forge_import::cost=debug` selects it for you). + --- ## 2. The Forge Mirror Action diff --git a/docs/guides/mirror-a-gitlab-project.md b/docs/guides/mirror-a-gitlab-project.md index 17686ce93..040747835 100644 --- a/docs/guides/mirror-a-gitlab-project.md +++ b/docs/guides/mirror-a-gitlab-project.md @@ -59,7 +59,7 @@ Start with a dry run. It reads everything, compares it with what the destination forge-import gitlab.com/acme/widget --max-spend 0.5 --state ./widget.sync.json ``` -The flags are the same as for GitHub: `--repo`, `--sync`, `--max-spend`, `--state`, `--dry-run`, `--limit`, `--yes`, `--summary-json`, and the network flags. See the [GitHub guide's table](mirror-a-github-repo.md#1-first-import). In `--sync`, `prs` means merge requests (`mrs` works too). `dg import gitlab.com/acme/widget` is the same engine. +The flags are the same as for GitHub: `--repo`, `--sync`, `--max-spend`, `--state`, `--dry-run`, `--limit`, `--concurrency`, `--yes`, `--summary-json`, and the network flags. See the [GitHub guide's table](mirror-a-github-repo.md#1-first-import). In `--sync`, `prs` means merge requests (`mrs` works too). `dg import gitlab.com/acme/widget` is the same engine. **Re-running is safe and cheap.** What is already mirrored is decided on chain, by the GitLab URL each document records (`https://gitlab.com/acme/widget/-/issues/12`, `…/-/merge_requests/3`, and `…#note_` for a comment). A re-run writes only what is new and costs nothing when nothing changed. It is the same scheme as the GitHub importer.