From be506734b3ed3775f62c7060ad4fdfde98b6d4d5 Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Wed, 12 Aug 2026 09:44:46 +0800 Subject: [PATCH 1/9] fix(l1): bump EIP-1559 fees on same-nonce poster retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Track in-flight fees and floor re-estimates to the shared ≥10% replacement rule so flat markets cannot underprice replacements. --- sequencer/src/l1/eip1559.rs | 118 ++++++++++++++++++++++++++- sequencer/src/l1/submitter/poster.rs | 46 +++++++++-- sequencer/src/recovery/flusher.rs | 64 ++------------- 3 files changed, 163 insertions(+), 65 deletions(-) diff --git a/sequencer/src/l1/eip1559.rs b/sequencer/src/l1/eip1559.rs index 99ec7173..4dba81f5 100644 --- a/sequencer/src/l1/eip1559.rs +++ b/sequencer/src/l1/eip1559.rs @@ -1,7 +1,10 @@ // (c) Cartesi and individual authors (see AUTHORS) // SPDX-License-Identifier: Apache-2.0 -//! Shared EIP-1559 fee estimate used by the poster and fee oracle. +//! Shared EIP-1559 fee estimate and same-nonce replacement bump. +//! +//! Used by the batch poster (submission + retry), the fee oracle (charge), and +//! the mempool flusher (no-op replacement). use alloy::consensus::BlockHeader; use alloy::providers::{DynProvider, Provider, utils}; @@ -15,6 +18,38 @@ pub struct Eip1559Fees { pub max_fee_per_gas: u128, } +/// Bump EIP-1559 fees for a same-nonce replacement under the ≥10% rule. +/// +/// `max_fee` gets ×1.1 (+1 for integer-rounding flat spots); priority doubles +/// (intentionally generous past the 10% threshold). The poster floors a +/// re-estimate against the last successful send at that wallet nonce; the +/// flusher bumps a fresh estimate so no-ops can compete with pending batch +/// txs. Eviction is operational acceleration, not a correctness precondition. +pub fn bumped_replacement_fees(base_max_fee: u128, base_priority_fee: u128) -> (u128, u128) { + let new_max_fee = base_max_fee.saturating_mul(11) / 10 + 1; + let new_priority_fee = base_priority_fee.saturating_mul(2).max(1); + (new_max_fee, new_priority_fee) +} + +/// Absolute estimate, raised to a replacement floor when `prior` is set. +/// +/// First send at a nonce uses `estimate` unchanged. A same-nonce resubmit +/// takes the per-component max of the fresh estimate and +/// [`bumped_replacement_fees`] of the last successful broadcast, so a flat +/// market cannot re-broadcast underpriced replacements. +pub fn fees_for_nonce(estimate: Eip1559Fees, prior: Option) -> Eip1559Fees { + let Some(prior) = prior else { + return estimate; + }; + let (bumped_max, bumped_prio) = + bumped_replacement_fees(prior.max_fee_per_gas, prior.max_priority_fee_per_gas); + Eip1559Fees { + base_fee_per_gas: estimate.base_fee_per_gas, + max_fee_per_gas: estimate.max_fee_per_gas.max(bumped_max), + max_priority_fee_per_gas: estimate.max_priority_fee_per_gas.max(bumped_prio), + } +} + /// Estimate fees with Alloy's default, MetaMask-style medium estimator. /// /// We intentionally pin the policy constants here: 10 historical blocks, the @@ -65,4 +100,85 @@ mod tests { assert_eq!(estimate.max_priority_fee_per_gas, 4); assert_eq!(estimate.max_fee_per_gas, 204); } + + #[test] + fn replacement_fee_bump_exceeds_ten_percent_for_max_fee() { + for base in [1_u128, 10, 100, 1_000, 1_000_000, 1_000_000_000_000] { + let (new_max, _) = bumped_replacement_fees(base, 0); + assert!( + new_max.saturating_mul(10) >= base.saturating_mul(11), + "max_fee bump violates ≥10% rule: base={base}, new={new_max}", + ); + } + } + + #[test] + fn replacement_fee_bump_doubles_priority_fee() { + for base in [1_u128, 10, 1_000, 1_000_000_000] { + let (_, new_prio) = bumped_replacement_fees(0, base); + assert_eq!(new_prio, base.saturating_mul(2)); + assert!( + new_prio.saturating_mul(10) >= base.saturating_mul(11), + "priority bump violates ≥10% rule: base={base}, new={new_prio}", + ); + } + } + + #[test] + fn replacement_fee_floor_is_positive_even_when_base_is_zero() { + let (new_max, new_prio) = bumped_replacement_fees(0, 0); + assert!(new_max >= 1); + assert!(new_prio >= 1); + } + + #[test] + fn replacement_fee_bump_saturates_at_u128_max() { + let (new_max, new_prio) = bumped_replacement_fees(u128::MAX, u128::MAX); + assert_eq!(new_max, u128::MAX / 10 + 1); + assert_eq!(new_prio, u128::MAX); + } + + #[test] + fn fees_for_nonce_passes_estimate_through_on_first_send() { + let estimate = Eip1559Fees { + base_fee_per_gas: 100, + max_priority_fee_per_gas: 2, + max_fee_per_gas: 202, + }; + assert_eq!(fees_for_nonce(estimate, None), estimate); + } + + #[test] + fn fees_for_nonce_floors_flat_estimate_to_replacement_bump() { + let prior = Eip1559Fees { + base_fee_per_gas: 100, + max_priority_fee_per_gas: 10, + max_fee_per_gas: 1_000, + }; + // Flat market: estimate equals prior. Replacement must clear ≥10%. + let estimate = prior; + let fees = fees_for_nonce(estimate, Some(prior)); + let (bumped_max, bumped_prio) = + bumped_replacement_fees(prior.max_fee_per_gas, prior.max_priority_fee_per_gas); + assert_eq!(fees.max_fee_per_gas, bumped_max); + assert_eq!(fees.max_priority_fee_per_gas, bumped_prio); + assert!(fees.max_fee_per_gas > prior.max_fee_per_gas); + assert!(fees.max_priority_fee_per_gas > prior.max_priority_fee_per_gas); + } + + #[test] + fn fees_for_nonce_keeps_estimate_when_market_already_clears_bump() { + let prior = Eip1559Fees { + base_fee_per_gas: 100, + max_priority_fee_per_gas: 10, + max_fee_per_gas: 1_000, + }; + let estimate = Eip1559Fees { + base_fee_per_gas: 500, + max_priority_fee_per_gas: 50, + max_fee_per_gas: 10_000, + }; + let fees = fees_for_nonce(estimate, Some(prior)); + assert_eq!(fees, estimate); + } } diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index ff451b60..ff4f3ad4 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -12,9 +12,11 @@ use sequencer_core::batch::Batch; use thiserror::Error; use tracing::{debug, info, warn}; -use crate::l1::eip1559::{Eip1559Fees, estimate_fees}; +use crate::l1::eip1559::{Eip1559Fees, estimate_fees, fees_for_nonce}; use crate::l1::partition::{decode_evm_advance_input, get_input_added_events_ordered}; use crate::l1::watermark::WalletNonceWatermarkSink; +use std::collections::BTreeMap; +use std::sync::{Arc, Mutex}; pub type TxHash = alloy_primitives::B256; @@ -68,11 +70,20 @@ pub trait BatchPoster: Send + Sync { pub struct EthereumBatchPoster { provider: DynProvider, config: BatchPosterConfig, + /// Fees of the last successful broadcast per wallet nonce still ≥ Latest. + /// Same-nonce retries floor a fresh estimate against + /// [`crate::l1::eip1559::bumped_replacement_fees`] of this record so a + /// flat market cannot re-broadcast underpriced replacements. + in_flight_fees: Arc>>, } impl EthereumBatchPoster { pub fn new(provider: DynProvider, config: BatchPosterConfig) -> Self { - Self { provider, config } + Self { + provider, + config, + in_flight_fees: Arc::new(Mutex::new(BTreeMap::new())), + } } /// Conservative upper-bound timeout for waiting on confirmations, derived @@ -123,11 +134,11 @@ impl EthereumBatchPoster { /// the time we start watching it. /// /// Timeouts return `Ok(())` rather than `Err` because the safe response is - /// "re-enter `submit_batches` on the next tick" — which re-estimates fees - /// (possibly replacing a pending transaction if the node accepts it) and - /// re-submits at the same wallet nonces. The - /// wallet-nonce ordering invariant above guarantees we cannot accidentally - /// skip work by returning early here. + /// "re-enter `submit_batches` on the next tick" — which re-estimates fees, + /// floors them to an explicit ≥10% replacement bump against any still + /// in-flight same-nonce submission, and re-submits at the same wallet + /// nonces. The wallet-nonce ordering invariant above guarantees we cannot + /// accidentally skip work by returning early here. async fn wait_for_confirmations(&self, tx_hashes: &[TxHash]) -> Result<(), BatchPosterError> { let timeout = self.confirmation_timeout(); for tx_hash in tx_hashes { @@ -203,11 +214,18 @@ impl BatchPoster for EthereumBatchPoster { }); } - let fees = estimate_fees(&self.provider) + let estimate = estimate_fees(&self.provider) .await .map_err(BatchPosterError::Provider)?; let mut next_nonce = self.latest_account_nonce().await?; + // Drop fee floors for nonces Latest has advanced past — those slots + // are resolved and must not floor a later send. + { + let mut in_flight = self.in_flight_fees.lock().expect("in_flight_fees lock"); + in_flight.retain(|&nonce, _| nonce >= next_nonce); + } + // Write-before-broadcast (R1a): durably cover every nonce this // tick will use before the first send. One raise to the highest // covers the whole consecutive range. @@ -219,11 +237,23 @@ impl BatchPoster for EthereumBatchPoster { let mut tx_hashes = Vec::with_capacity(payloads.len()); for payload in payloads { + let fees = { + let in_flight = self.in_flight_fees.lock().expect("in_flight_fees lock"); + fees_for_nonce(estimate, in_flight.get(&next_nonce).copied()) + }; let pending = self.send_batch_at_nonce(payload, next_nonce, &fees).await?; + // Record only after a successful broadcast — a failed send must + // not raise the replacement floor for the next tick. + self.in_flight_fees + .lock() + .expect("in_flight_fees lock") + .insert(next_nonce, fees); let tx_hash = *pending.tx_hash(); debug!( tx_nonce = next_nonce, %tx_hash, + max_fee_per_gas = fees.max_fee_per_gas, + max_priority_fee_per_gas = fees.max_priority_fee_per_gas, confirmation_depth = self.config.confirmation_depth, "sent batch submission tx to L1" ); diff --git a/sequencer/src/recovery/flusher.rs b/sequencer/src/recovery/flusher.rs index e7de31f3..e6d925ac 100644 --- a/sequencer/src/recovery/flusher.rs +++ b/sequencer/src/recovery/flusher.rs @@ -19,6 +19,7 @@ use std::time::Duration; use thiserror::Error; use tracing::{debug, error, info}; +use crate::l1::eip1559::bumped_replacement_fees; use crate::l1::watermark::{StorageWatermarkSink, WalletNonceWatermarkSink}; #[derive(Debug, Error)] @@ -50,20 +51,6 @@ fn derive_timeouts(seconds_per_block: u64) -> (Duration, Duration) { ) } -/// Bump current 1559 fee estimates so flush no-ops are competitive with -/// pending batch transactions at the same wallet nonces. -/// -/// Safety does not depend on the no-op winning. Either the original batch tx -/// or the no-op can consume the slot; `flush_and_wait` only returns once -/// `Pending <= Safe`. These bumped fees are an operational acceleration, not a -/// correctness precondition. The `+ 1` on `max_fee` avoids integer-rounding -/// flat spots, and the priority doubling is intentionally generous. -fn bumped_replacement_fees(base_max_fee: u128, base_priority_fee: u128) -> (u128, u128) { - let new_max_fee = base_max_fee.saturating_mul(11) / 10 + 1; - let new_priority_fee = base_priority_fee.saturating_mul(2).max(1); - (new_max_fee, new_priority_fee) -} - fn send_failures_error(failures: &[(u64, String)]) -> FlushError { const MAX_SAMPLES: usize = 3; @@ -258,6 +245,10 @@ impl MempoolFlusher { .await .map_err(|e| FlushError::Provider(e.to_string()))?; + // Bump the absolute estimate so no-ops can compete with pending batch + // txs at the same wallet nonces (shared ≥10% replacement rule). Safety + // does not depend on the no-op winning — `flush_and_wait` only returns + // once Pending ≤ Safe. let (bumped_max_fee, bumped_priority_fee) = bumped_replacement_fees(fees.max_fee_per_gas, fees.max_priority_fee_per_gas); @@ -369,40 +360,9 @@ mod tests { } // ── H5: replacement-fee bump keeps no-ops competitive ───────── - - #[test] - fn replacement_fee_bump_exceeds_ten_percent_for_max_fee() { - // `max_fee_per_gas` must strictly exceed base by ≥10% for any positive base. - for base in [1_u128, 10, 100, 1_000, 1_000_000, 1_000_000_000_000] { - let (new_max, _) = bumped_replacement_fees(base, 0); - assert!( - new_max.saturating_mul(10) >= base.saturating_mul(11), - "max_fee bump violates ≥10% rule: base={base}, new={new_max}", - ); - } - } - - #[test] - fn replacement_fee_bump_doubles_priority_fee() { - // `priority_fee` doubles (200%), easily clearing the 10% replacement threshold. - for base in [1_u128, 10, 1_000, 1_000_000_000] { - let (_, new_prio) = bumped_replacement_fees(0, base); - assert_eq!(new_prio, base.saturating_mul(2)); - assert!( - new_prio.saturating_mul(10) >= base.saturating_mul(11), - "priority bump violates ≥10% rule: base={base}, new={new_prio}", - ); - } - } - - #[test] - fn replacement_fee_floor_is_positive_even_when_base_is_zero() { - // If the estimator returns zero, bumped values are still positive so the - // tx is actually broadcast rather than rejected by the node. - let (new_max, new_prio) = bumped_replacement_fees(0, 0); - assert!(new_max >= 1); - assert!(new_prio >= 1); - } + // Rule itself lives in `l1::eip1559` (shared with the poster); the + // flusher's use site is the `bumped_replacement_fees(...)` call in + // `submit_noops`. #[test] fn send_failure_error_summarizes_failed_slots() { @@ -433,14 +393,6 @@ mod tests { assert!(matches!(err, FlushError::Provider(_))); } - #[test] - fn replacement_fee_bump_saturates_at_u128_max() { - // Overflow safety: astronomical base fees must not wrap around. - let (new_max, new_prio) = bumped_replacement_fees(u128::MAX, u128::MAX); - assert_eq!(new_max, u128::MAX / 10 + 1); - assert_eq!(new_prio, u128::MAX); - } - // ── H6: timeouts derive from seconds_per_block ──────────────── #[test] From 77a4a61042a84863fe43a0609981d29bac310003 Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Wed, 12 Aug 2026 10:03:48 +0800 Subject: [PATCH 2/9] test(l1): cover poster in-flight fee floor wiring Add mixed-component fees_for_nonce coverage, seeded replacement floor, Latest prune, and failed-send non-record assertions. --- sequencer/src/l1/eip1559.rs | 36 +++++ sequencer/src/l1/submitter/poster.rs | 197 +++++++++++++++++++++++++++ 2 files changed, 233 insertions(+) diff --git a/sequencer/src/l1/eip1559.rs b/sequencer/src/l1/eip1559.rs index 4dba81f5..8a5244b6 100644 --- a/sequencer/src/l1/eip1559.rs +++ b/sequencer/src/l1/eip1559.rs @@ -181,4 +181,40 @@ mod tests { let fees = fees_for_nonce(estimate, Some(prior)); assert_eq!(fees, estimate); } + + #[test] + fn fees_for_nonce_clears_both_fields_when_estimate_is_mixed() { + // Market moved up on max_fee but not on priority (or the reverse): + // each component must still clear the ≥10% floor vs the in-flight tx. + let prior = Eip1559Fees { + base_fee_per_gas: 100, + max_priority_fee_per_gas: 100, + max_fee_per_gas: 1_000, + }; + let (bumped_max, bumped_prio) = + bumped_replacement_fees(prior.max_fee_per_gas, prior.max_priority_fee_per_gas); + + // High max_fee estimate, priority still below the replacement floor. + let estimate_high_max = Eip1559Fees { + base_fee_per_gas: 100, + max_priority_fee_per_gas: prior.max_priority_fee_per_gas + 1, // < 10% bump + max_fee_per_gas: bumped_max + 5_000, + }; + let fees = fees_for_nonce(estimate_high_max, Some(prior)); + assert_eq!(fees.max_fee_per_gas, estimate_high_max.max_fee_per_gas); + assert_eq!(fees.max_priority_fee_per_gas, bumped_prio); + + // High priority estimate, max_fee still below the replacement floor. + let estimate_high_prio = Eip1559Fees { + base_fee_per_gas: 100, + max_priority_fee_per_gas: bumped_prio + 50, + max_fee_per_gas: prior.max_fee_per_gas + 1, // < 10% bump + }; + let fees = fees_for_nonce(estimate_high_prio, Some(prior)); + assert_eq!(fees.max_fee_per_gas, bumped_max); + assert_eq!( + fees.max_priority_fee_per_gas, + estimate_high_prio.max_priority_fee_per_gas + ); + } } diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index ff4f3ad4..38ab26ff 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -16,6 +16,8 @@ use crate::l1::eip1559::{Eip1559Fees, estimate_fees, fees_for_nonce}; use crate::l1::partition::{decode_evm_advance_input, get_input_added_events_ordered}; use crate::l1::watermark::WalletNonceWatermarkSink; use std::collections::BTreeMap; +#[cfg(test)] +use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; pub type TxHash = alloy_primitives::B256; @@ -75,6 +77,10 @@ pub struct EthereumBatchPoster { /// [`crate::l1::eip1559::bumped_replacement_fees`] of this record so a /// flat market cannot re-broadcast underpriced replacements. in_flight_fees: Arc>>, + /// Test-only: next `send_batch_at_nonce` returns Err without broadcasting, + /// so callers can assert the in-flight map is not updated on send failure. + #[cfg(test)] + fail_next_send: Arc, } impl EthereumBatchPoster { @@ -83,9 +89,29 @@ impl EthereumBatchPoster { provider, config, in_flight_fees: Arc::new(Mutex::new(BTreeMap::new())), + #[cfg(test)] + fail_next_send: Arc::new(AtomicBool::new(false)), } } + #[cfg(test)] + pub(crate) fn in_flight_fees_for_test(&self) -> BTreeMap { + self.in_flight_fees + .lock() + .expect("in_flight_fees lock") + .clone() + } + + #[cfg(test)] + pub(crate) fn seed_in_flight_fees_for_test(&self, fees: BTreeMap) { + *self.in_flight_fees.lock().expect("in_flight_fees lock") = fees; + } + + #[cfg(test)] + pub(crate) fn fail_next_send_for_test(&self) { + self.fail_next_send.store(true, Ordering::SeqCst); + } + /// Conservative upper-bound timeout for waiting on confirmations, derived /// from the configured block time. Shorter block times on other chains just /// make the watch complete sooner. @@ -110,6 +136,14 @@ impl EthereumBatchPoster { nonce: u64, fees: &Eip1559Fees, ) -> Result, BatchPosterError> { + #[cfg(test)] + { + if self.fail_next_send.swap(false, Ordering::SeqCst) { + return Err(BatchPosterError::Provider( + "test-injected send failure".to_string(), + )); + } + } let input_box = InputBox::new(self.config.l1_submit_address, &self.provider); input_box .addInput(self.config.app_address, payload.into()) @@ -404,6 +438,7 @@ pub(crate) mod mock { #[cfg(test)] mod tests { + use std::collections::BTreeMap; use std::sync::Mutex; use std::time::Duration; @@ -609,4 +644,166 @@ mod tests { assert_eq!(derive_confirmation_timeout(2, 1), Duration::from_secs(6)); assert_eq!(derive_confirmation_timeout(5, 3), Duration::from_secs(36)); } + + fn poster_config(anvil: &alloy::node_bindings::AnvilInstance) -> BatchPosterConfig { + BatchPosterConfig { + l1_submit_address: alloy_primitives::Address::repeat_byte(0x11), + app_address: alloy_primitives::Address::repeat_byte(0x22), + batch_submitter_address: alloy_primitives::address!( + "0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266" + ), + start_block: 0, + // confirmation_depth 0 → watch timeout is 2 * seconds_per_block; + // keep it short so --no-mining ticks return promptly on timeout. + confirmation_depth: 0, + seconds_per_block: 1, + long_block_range_error_codes: vec![], + expected_chain_id: anvil.chain_id(), + } + } + + /// Same-nonce retry floors a flat re-estimate against the in-flight record + /// (≥10% on both fields). Seeds the prior floor explicitly so the assertion + /// does not depend on Anvil keeping a tx pending across ticks. + #[tokio::test] + async fn submit_batches_replacement_clears_ten_percent_bump() { + require_anvil(); + let anvil = Anvil::default().timeout(30_000).spawn(); + let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + let submitter = alloy_primitives::address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); + let provider = crate::l1::provider::create_signer_provider(&anvil.endpoint(), key, false) + .expect("signer provider"); + let poster = EthereumBatchPoster::new(provider.clone(), poster_config(&anvil)); + let sink = RecordingWatermarkSink::passing(); + + let base_nonce = provider + .get_transaction_count(submitter) + .await + .expect("base nonce"); + // Prior fees high enough that a fresh Anvil estimate will not clear the + // ≥10% floor on its own — the poster must bump against this record. + let prior = crate::l1::eip1559::Eip1559Fees { + base_fee_per_gas: 1, + max_priority_fee_per_gas: 50_000_000, // 0.05 gwei + max_fee_per_gas: 100_000_000_000, // 100 gwei + }; + poster.seed_in_flight_fees_for_test(BTreeMap::from([(base_nonce, prior)])); + + poster + .submit_batches(vec![vec![0u8; 4]], &sink) + .await + .expect("submit with in-flight floor"); + let sent = poster + .in_flight_fees_for_test() + .get(&base_nonce) + .copied() + .expect("successful send must record fees"); + + let (bumped_max, bumped_prio) = crate::l1::eip1559::bumped_replacement_fees( + prior.max_fee_per_gas, + prior.max_priority_fee_per_gas, + ); + assert!( + sent.max_fee_per_gas >= bumped_max, + "max_fee must clear replacement floor: sent={} floor={bumped_max}", + sent.max_fee_per_gas + ); + assert!( + sent.max_priority_fee_per_gas >= bumped_prio, + "priority must clear replacement floor: sent={} floor={bumped_prio}", + sent.max_priority_fee_per_gas + ); + } + + /// When Latest advances past a nonce, that nonce's fee floor is dropped so a + /// later tip send is not incorrectly floored by stale in-flight state. + #[tokio::test] + async fn submit_batches_prunes_in_flight_fees_past_latest() { + require_anvil(); + let anvil = Anvil::default().timeout(30_000).spawn(); // automine on + let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + let submitter = alloy_primitives::address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); + let provider = crate::l1::provider::create_signer_provider(&anvil.endpoint(), key, false) + .expect("signer provider"); + let poster = EthereumBatchPoster::new(provider.clone(), poster_config(&anvil)); + let sink = RecordingWatermarkSink::passing(); + + let base_nonce = provider + .get_transaction_count(submitter) + .await + .expect("base nonce"); + + poster + .submit_batches(vec![vec![0u8; 4]], &sink) + .await + .expect("first submit mines under automine"); + assert!( + poster.in_flight_fees_for_test().contains_key(&base_nonce), + "first send records fees for the mined nonce" + ); + + // Tip confirmed → Latest = base_nonce + 1. Re-seed a stale floor on the + // mined nonce (as if a previous tick left it) and confirm the next + // submit prunes it. + let stale = crate::l1::eip1559::Eip1559Fees { + base_fee_per_gas: 1, + max_priority_fee_per_gas: 1, + max_fee_per_gas: 1, + }; + poster.seed_in_flight_fees_for_test(BTreeMap::from([(base_nonce, stale)])); + + poster + .submit_batches(vec![vec![1u8; 4]], &sink) + .await + .expect("second submit"); + + let in_flight = poster.in_flight_fees_for_test(); + assert!( + !in_flight.contains_key(&base_nonce), + "mined nonce must be pruned once Latest advances: {in_flight:?}" + ); + let tip_nonce = base_nonce.saturating_add(1); + assert!( + in_flight.contains_key(&tip_nonce), + "current tip send must be recorded: {in_flight:?}" + ); + } + + /// A failed broadcast must not raise the replacement floor — otherwise a + /// blip would permanently overprice the next successful send, or worse, + /// record fees for a tx that never entered the mempool. + #[tokio::test] + async fn submit_batches_does_not_record_fees_when_send_fails() { + require_anvil(); + let anvil = Anvil::default().arg("--no-mining").timeout(30_000).spawn(); + let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + let submitter = alloy_primitives::address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); + let provider = crate::l1::provider::create_signer_provider(&anvil.endpoint(), key, false) + .expect("signer provider"); + let poster = EthereumBatchPoster::new(provider.clone(), poster_config(&anvil)); + let sink = RecordingWatermarkSink::passing(); + + let base_nonce = provider + .get_transaction_count(submitter) + .await + .expect("base nonce"); + let prior = crate::l1::eip1559::Eip1559Fees { + base_fee_per_gas: 42, + max_priority_fee_per_gas: 7, + max_fee_per_gas: 1_000, + }; + poster.seed_in_flight_fees_for_test(BTreeMap::from([(base_nonce, prior)])); + poster.fail_next_send_for_test(); + + let result = poster.submit_batches(vec![vec![0u8; 4]], &sink).await; + assert!( + matches!(result, Err(BatchPosterError::Provider(ref msg)) if msg.contains("test-injected")), + "injected send failure must surface, got {result:?}" + ); + assert_eq!( + poster.in_flight_fees_for_test(), + BTreeMap::from([(base_nonce, prior)]), + "failed send must leave the prior in-flight floor untouched" + ); + } } From e54879bd8127d86b845377b5b90cfc71486c8d8e Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Tue, 18 Aug 2026 22:03:12 +0800 Subject: [PATCH 3/9] test(l1): unflake fee-oracle retain-across-transient loop A 200ms sleep raced the first spawn_blocking SQLite write on loaded CI, so the assertion still saw the default log_gas_price of 0. --- sequencer/src/l1/fee_oracle/worker.rs | 43 ++++++++++++++++++++------- 1 file changed, 32 insertions(+), 11 deletions(-) diff --git a/sequencer/src/l1/fee_oracle/worker.rs b/sequencer/src/l1/fee_oracle/worker.rs index 06c0322e..fe98823f 100644 --- a/sequencer/src/l1/fee_oracle/worker.rs +++ b/sequencer/src/l1/fee_oracle/worker.rs @@ -277,7 +277,8 @@ mod tests { use super::*; use crate::storage::test_helpers::temp_db; use alloy_primitives::U256; - use std::sync::Mutex; + use std::sync::atomic::{AtomicUsize, Ordering}; + use std::sync::{Arc, Mutex}; const TEST_MAX_AGE_MS: u64 = 60 * 60 * 1000; @@ -300,16 +301,15 @@ mod tests { } struct FailsAfterFirstGas { - calls: Mutex, + calls: Arc, ok: Eip1559Fees, } #[async_trait] impl GasFeeSource for FailsAfterFirstGas { async fn estimate_gas_fees(&self) -> Result { - let mut calls = self.calls.lock().expect("lock"); - *calls += 1; - if *calls == 1 { + let n = self.calls.fetch_add(1, Ordering::SeqCst) + 1; + if n == 1 { Ok(self.ok) } else { Err("rpc unavailable".into()) @@ -438,7 +438,7 @@ mod tests { &db.path, TEST_MAX_AGE_MS, Box::new(FailsAfterFirstGas { - calls: Mutex::new(0), + calls: Arc::new(AtomicUsize::new(0)), ok: sample_fees(), }), Box::new(StaticToken(sample_quote())), @@ -532,12 +532,13 @@ mod tests { initialize_db(&db.path); let expected_log = expected_log_price(); + let gas_calls = Arc::new(AtomicUsize::new(0)); let oracle = FeeOracle::new_with_sources( db.path.clone(), Duration::from_millis(40), TEST_MAX_AGE_MS, Box::new(FailsAfterFirstGas { - calls: Mutex::new(0), + calls: Arc::clone(&gas_calls), ok: sample_fees(), }), Box::new(StaticToken(sample_quote())), @@ -545,10 +546,30 @@ mod tests { let shutdown = ShutdownSignal::default(); let mut handle = oracle.start(shutdown.clone()); - tokio::select! { - biased; - result = &mut handle => panic!("fee oracle exited early: {result:?}"), - _ = tokio::time::sleep(Duration::from_millis(200)) => {} + // First refresh is a spawn_blocking SQLite write; a fixed sleep flakes + // when the blocking pool is busy. Wait until the price is persisted + // *and* a later tick has failed, so retain-on-transient is covered. + let deadline = tokio::time::Instant::now() + Duration::from_secs(2); + loop { + tokio::select! { + biased; + result = &mut handle => panic!("fee oracle exited early: {result:?}"), + _ = tokio::time::sleep(Duration::from_millis(10)) => {} + } + let price = Storage::open_read_only(&db.path) + .unwrap() + .log_gas_price() + .unwrap(); + let calls = gas_calls.load(Ordering::SeqCst); + if price == expected_log && calls >= 2 { + break; + } + if tokio::time::Instant::now() >= deadline { + panic!( + "timed out waiting for retained fee-oracle price \ + (expected {expected_log}, last read {price}, gas_calls={calls})" + ); + } } let storage = Storage::open_read_only(&db.path).unwrap(); assert_eq!(storage.log_gas_price().unwrap(), expected_log); From ba0275082ad8337e71d8c31edf32c9621885980a Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:34:14 +0800 Subject: [PATCH 4/9] fix(l1): keep replacement bumps EIP-1559-valid across retries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Grow max_fee and priority by the same ×1.1+1 factor and clamp tip ≤ cap so poster retries cannot wedge on ErrTipAboveFeeCap. Only the blocking (Latest) nonce is re-escalated; suffix txs keep their original hash. --- sequencer/src/l1/eip1559.rs | 163 +++++++++++++++++++++++---- sequencer/src/l1/submitter/poster.rs | 132 ++++++++++++++++++---- sequencer/src/recovery/flusher.rs | 14 ++- 3 files changed, 261 insertions(+), 48 deletions(-) diff --git a/sequencer/src/l1/eip1559.rs b/sequencer/src/l1/eip1559.rs index 8a5244b6..147300b3 100644 --- a/sequencer/src/l1/eip1559.rs +++ b/sequencer/src/l1/eip1559.rs @@ -18,35 +18,62 @@ pub struct Eip1559Fees { pub max_fee_per_gas: u128, } +/// Bump one EIP-1559 component for a same-nonce replacement. +/// +/// ×1.1, plus 1 wei so integer division cannot stall on a flat spot and so +/// geth's strict-greater precheck still passes when `x` is tiny. Saturating +/// `x+1` keeps a `u128::MAX` fee from shrinking after `saturating_mul`. +fn bump_replacement_component(value: u128) -> u128 { + let bumped = value.saturating_mul(11) / 10 + 1; + bumped.max(value.saturating_add(1)) +} + /// Bump EIP-1559 fees for a same-nonce replacement under the ≥10% rule. /// -/// `max_fee` gets ×1.1 (+1 for integer-rounding flat spots); priority doubles -/// (intentionally generous past the 10% threshold). The poster floors a -/// re-estimate against the last successful send at that wallet nonce; the -/// flusher bumps a fresh estimate so no-ops can compete with pending batch -/// txs. Eviction is operational acceleration, not a correctness precondition. +/// Both `max_fee` and the priority tip grow by the same ×1.1 (+1) factor. +/// Asymmetric growth (tip ×2, cap ×1.1) compounds across poster retries until +/// `tip > max_fee` — an invalid EIP-1559 tx every node rejects +/// (`ErrTipAboveFeeCap`), and because a failed send does not update the +/// in-flight floor the poster then resubmits the identical invalid pair +/// forever. Equal growth preserves `tip ≤ max_fee` whenever the input did; +/// the clamp is defense-in-depth for a one-shot bump of a near-zero-base +/// estimate (the flusher) and for already-invalid inputs. +/// +/// The poster floors a re-estimate against the last successful send at that +/// wallet nonce; the flusher bumps a fresh estimate so no-ops can compete +/// with pending batch txs. Eviction is operational acceleration, not a +/// correctness precondition. pub fn bumped_replacement_fees(base_max_fee: u128, base_priority_fee: u128) -> (u128, u128) { - let new_max_fee = base_max_fee.saturating_mul(11) / 10 + 1; - let new_priority_fee = base_priority_fee.saturating_mul(2).max(1); + let tip = base_priority_fee.min(base_max_fee); + let new_max_fee = bump_replacement_component(base_max_fee); + let new_priority_fee = bump_replacement_component(tip).min(new_max_fee); (new_max_fee, new_priority_fee) } /// Absolute estimate, raised to a replacement floor when `prior` is set. /// -/// First send at a nonce uses `estimate` unchanged. A same-nonce resubmit -/// takes the per-component max of the fresh estimate and +/// First send at a nonce uses `estimate` (clamped so `tip ≤ max_fee`). A +/// same-nonce resubmit takes the per-component max of the fresh estimate and /// [`bumped_replacement_fees`] of the last successful broadcast, so a flat -/// market cannot re-broadcast underpriced replacements. +/// market cannot re-broadcast underpriced replacements. The tip is then +/// clamped to the fee cap: geth will not accept `maxPriorityFeePerGas > +/// maxFeePerGas`, and a rejected send must not become a sticky invalid pair. pub fn fees_for_nonce(estimate: Eip1559Fees, prior: Option) -> Eip1559Fees { - let Some(prior) = prior else { - return estimate; + let fees = match prior { + None => estimate, + Some(prior) => { + let (bumped_max, bumped_prio) = + bumped_replacement_fees(prior.max_fee_per_gas, prior.max_priority_fee_per_gas); + Eip1559Fees { + base_fee_per_gas: estimate.base_fee_per_gas, + max_fee_per_gas: estimate.max_fee_per_gas.max(bumped_max), + max_priority_fee_per_gas: estimate.max_priority_fee_per_gas.max(bumped_prio), + } + } }; - let (bumped_max, bumped_prio) = - bumped_replacement_fees(prior.max_fee_per_gas, prior.max_priority_fee_per_gas); Eip1559Fees { - base_fee_per_gas: estimate.base_fee_per_gas, - max_fee_per_gas: estimate.max_fee_per_gas.max(bumped_max), - max_priority_fee_per_gas: estimate.max_priority_fee_per_gas.max(bumped_prio), + max_priority_fee_per_gas: fees.max_priority_fee_per_gas.min(fees.max_fee_per_gas), + ..fees } } @@ -113,14 +140,33 @@ mod tests { } #[test] - fn replacement_fee_bump_doubles_priority_fee() { - for base in [1_u128, 10, 1_000, 1_000_000_000] { - let (_, new_prio) = bumped_replacement_fees(0, base); - assert_eq!(new_prio, base.saturating_mul(2)); + fn replacement_fee_bump_exceeds_ten_percent_for_priority_fee() { + for base in [1_u128, 10, 100, 1_000, 1_000_000, 1_000_000_000_000] { + // Cap high enough that the tip clamp does not bind. + let (_, new_prio) = bumped_replacement_fees(base.saturating_mul(4), base); assert!( new_prio.saturating_mul(10) >= base.saturating_mul(11), "priority bump violates ≥10% rule: base={base}, new={new_prio}", ); + assert!(new_prio > base); + } + } + + #[test] + fn replacement_fee_bump_keeps_tip_at_or_below_fee_cap() { + for (max_fee, tip) in [ + (0_u128, 0), + (0, 100), + (1, 1), + (1, 10), + (20_000_000_000, 1_000_000_000), + (u128::MAX, u128::MAX), + ] { + let (new_max, new_prio) = bumped_replacement_fees(max_fee, tip); + assert!( + new_prio <= new_max, + "bumped tip {new_prio} exceeds fee cap {new_max} (from max={max_fee} tip={tip})", + ); } } @@ -134,7 +180,7 @@ mod tests { #[test] fn replacement_fee_bump_saturates_at_u128_max() { let (new_max, new_prio) = bumped_replacement_fees(u128::MAX, u128::MAX); - assert_eq!(new_max, u128::MAX / 10 + 1); + assert_eq!(new_max, u128::MAX); assert_eq!(new_prio, u128::MAX); } @@ -217,4 +263,77 @@ mod tests { estimate_high_prio.max_priority_fee_per_gas ); } + + fn assert_eip1559_valid(fees: Eip1559Fees) { + assert!( + fees.max_priority_fee_per_gas <= fees.max_fee_per_gas, + "invalid EIP-1559 pair: tip {} > max_fee {}", + fees.max_priority_fee_per_gas, + fees.max_fee_per_gas, + ); + } + + fn assert_clears_replacement_floor(prior: Eip1559Fees, next: Eip1559Fees) { + assert!(next.max_fee_per_gas > prior.max_fee_per_gas); + assert!(next.max_priority_fee_per_gas > prior.max_priority_fee_per_gas); + assert!( + next.max_fee_per_gas.saturating_mul(10) >= prior.max_fee_per_gas.saturating_mul(11), + "max_fee lost the ≥10% floor: prior={} next={}", + prior.max_fee_per_gas, + next.max_fee_per_gas, + ); + assert!( + next.max_priority_fee_per_gas.saturating_mul(10) + >= prior.max_priority_fee_per_gas.saturating_mul(11), + "priority lost the ≥10% floor: prior={} next={}", + prior.max_priority_fee_per_gas, + next.max_priority_fee_per_gas, + ); + } + + #[test] + fn fees_for_nonce_stays_valid_across_repeated_flat_retries() { + // The poster records the *sent* pair, so a stuck nonce compounds the + // bump against its own output. Asymmetric ×2 tip / ×1.1 cap crossed + // `tip > max_fee` in ~7 rounds at 20 gwei base / 1 gwei tip — and on + // the first retry when cap ≈ tip. Equal growth must stay valid. + for start in [ + Eip1559Fees { + base_fee_per_gas: 20_000_000_000, + max_priority_fee_per_gas: 1_000_000_000, + max_fee_per_gas: 41_000_000_000, + }, + Eip1559Fees { + base_fee_per_gas: 1, + max_priority_fee_per_gas: 1, + max_fee_per_gas: 1, + }, + Eip1559Fees { + base_fee_per_gas: 0, + max_priority_fee_per_gas: 1_000, + max_fee_per_gas: 1_000, + }, + ] { + let mut fees = start; + assert_eip1559_valid(fees); + for _ in 0..20 { + let next = fees_for_nonce(fees, Some(fees)); + assert_eip1559_valid(next); + assert_clears_replacement_floor(fees, next); + fees = next; + } + } + } + + #[test] + fn fees_for_nonce_clamps_tip_above_fee_cap_on_first_send() { + let estimate = Eip1559Fees { + base_fee_per_gas: 1, + max_priority_fee_per_gas: 500, + max_fee_per_gas: 100, + }; + let fees = fees_for_nonce(estimate, None); + assert_eq!(fees.max_fee_per_gas, 100); + assert_eq!(fees.max_priority_fee_per_gas, 100); + } } diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index 38ab26ff..1b68fe89 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -22,6 +22,16 @@ use std::sync::{Arc, Mutex}; pub type TxHash = alloy_primitives::B256; +/// Last successful broadcast at a wallet nonce: fees we actually sent, and +/// the hash the next tick can keep watching if this nonce is no longer the +/// blocking head (so we do not replace it). `tx_hash` is `None` only in tests +/// that seed a fee floor without a prior send. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +struct InFlightTx { + fees: Eip1559Fees, + tx_hash: Option, +} + #[derive(Debug, Clone)] pub struct BatchPosterConfig { pub l1_submit_address: alloy_primitives::Address, @@ -72,11 +82,21 @@ pub trait BatchPoster: Send + Sync { pub struct EthereumBatchPoster { provider: DynProvider, config: BatchPosterConfig, - /// Fees of the last successful broadcast per wallet nonce still ≥ Latest. - /// Same-nonce retries floor a fresh estimate against - /// [`crate::l1::eip1559::bumped_replacement_fees`] of this record so a - /// flat market cannot re-broadcast underpriced replacements. - in_flight_fees: Arc>>, + /// Fees + tx hash of the last successful broadcast per wallet nonce still + /// ≥ Latest. + /// + /// Same-nonce retries of the **head** (Latest) nonce floor a fresh + /// estimate against [`crate::l1::eip1559::bumped_replacement_fees`] of + /// this record so a flat market cannot re-broadcast underpriced + /// replacements. Suffix nonces already in the map are left in the mempool: + /// only the head can be blocking, and re-escalating the whole unconfirmed + /// suffix compounds fees for txs that cannot mine until the head does. + /// + /// Process-local, so the floor is best-effort, not an invariant. A restart + /// (or a send whose response is lost after the node accepted) re-opens the + /// underpriced-retry window for a cycle. A future hardening is to also + /// raise the floor on a "replacement transaction underpriced" send error. + in_flight: Arc>>, /// Test-only: next `send_batch_at_nonce` returns Err without broadcasting, /// so callers can assert the in-flight map is not updated on send failure. #[cfg(test)] @@ -88,7 +108,7 @@ impl EthereumBatchPoster { Self { provider, config, - in_flight_fees: Arc::new(Mutex::new(BTreeMap::new())), + in_flight: Arc::new(Mutex::new(BTreeMap::new())), #[cfg(test)] fail_next_send: Arc::new(AtomicBool::new(false)), } @@ -96,15 +116,30 @@ impl EthereumBatchPoster { #[cfg(test)] pub(crate) fn in_flight_fees_for_test(&self) -> BTreeMap { - self.in_flight_fees + self.in_flight .lock() - .expect("in_flight_fees lock") - .clone() + .expect("in_flight lock") + .iter() + .map(|(&nonce, tx)| (nonce, tx.fees)) + .collect() } #[cfg(test)] pub(crate) fn seed_in_flight_fees_for_test(&self, fees: BTreeMap) { - *self.in_flight_fees.lock().expect("in_flight_fees lock") = fees; + let mut in_flight = self.in_flight.lock().expect("in_flight lock"); + in_flight.clear(); + for (nonce, fees) in fees { + in_flight.insert( + nonce, + InFlightTx { + fees, + // Tests that seed a floor then submit are replacing the + // head nonce; the hash is only used to skip suffix + // re-broadcast, which those tests do not exercise. + tx_hash: None, + }, + ); + } } #[cfg(test)] @@ -169,10 +204,11 @@ impl EthereumBatchPoster { /// /// Timeouts return `Ok(())` rather than `Err` because the safe response is /// "re-enter `submit_batches` on the next tick" — which re-estimates fees, - /// floors them to an explicit ≥10% replacement bump against any still - /// in-flight same-nonce submission, and re-submits at the same wallet - /// nonces. The wallet-nonce ordering invariant above guarantees we cannot - /// accidentally skip work by returning early here. + /// floors the **head** nonce to an explicit ≥10% replacement bump against + /// any still in-flight same-nonce submission, leaves already-broadcast + /// suffix txs in the mempool, and re-submits only what still needs a + /// replacement. The wallet-nonce ordering invariant above guarantees we + /// cannot accidentally skip work by returning early here. async fn wait_for_confirmations(&self, tx_hashes: &[TxHash]) -> Result<(), BatchPosterError> { let timeout = self.confirmation_timeout(); for tx_hash in tx_hashes { @@ -208,6 +244,15 @@ impl EthereumBatchPoster { } } +/// If this nonce is behind the blocking head and already in the mempool, keep +/// watching the original hash instead of replacing it. +fn suffix_watch_hash(head_nonce: u64, nonce: u64, existing: Option) -> Option { + if nonce == head_nonce { + return None; + } + existing.and_then(|tx| tx.tx_hash) +} + fn derive_confirmation_timeout( confirmation_depth: u64, seconds_per_block: u64, @@ -256,7 +301,7 @@ impl BatchPoster for EthereumBatchPoster { // Drop fee floors for nonces Latest has advanced past — those slots // are resolved and must not floor a later send. { - let mut in_flight = self.in_flight_fees.lock().expect("in_flight_fees lock"); + let mut in_flight = self.in_flight.lock().expect("in_flight lock"); in_flight.retain(|&nonce, _| nonce >= next_nonce); } @@ -269,20 +314,37 @@ impl BatchPoster for EthereumBatchPoster { .map_err(BatchPosterError::Provider)?; let mut tx_hashes = Vec::with_capacity(payloads.len()); + let head_nonce = next_nonce; for payload in payloads { - let fees = { - let in_flight = self.in_flight_fees.lock().expect("in_flight_fees lock"); - fees_for_nonce(estimate, in_flight.get(&next_nonce).copied()) + let existing = { + let in_flight = self.in_flight.lock().expect("in_flight lock"); + in_flight.get(&next_nonce).copied() }; + + if let Some(tx_hash) = suffix_watch_hash(head_nonce, next_nonce, existing) { + tx_hashes.push(tx_hash); + next_nonce = next_nonce.saturating_add(1); + continue; + } + + let prior = if next_nonce == head_nonce { + existing.map(|tx| tx.fees) + } else { + None + }; + let fees = fees_for_nonce(estimate, prior); let pending = self.send_batch_at_nonce(payload, next_nonce, &fees).await?; // Record only after a successful broadcast — a failed send must // not raise the replacement floor for the next tick. - self.in_flight_fees - .lock() - .expect("in_flight_fees lock") - .insert(next_nonce, fees); let tx_hash = *pending.tx_hash(); + self.in_flight.lock().expect("in_flight lock").insert( + next_nonce, + InFlightTx { + fees, + tx_hash: Some(tx_hash), + }, + ); debug!( tx_nonce = next_nonce, %tx_hash, @@ -443,8 +505,8 @@ mod tests { use std::time::Duration; use super::{ - BatchPoster, BatchPosterConfig, BatchPosterError, EthereumBatchPoster, - derive_confirmation_timeout, mock::MockBatchPoster, + BatchPoster, BatchPosterConfig, BatchPosterError, EthereumBatchPoster, InFlightTx, TxHash, + derive_confirmation_timeout, mock::MockBatchPoster, suffix_watch_hash, }; use crate::l1::watermark::WalletNonceWatermarkSink; use alloy::node_bindings::Anvil; @@ -645,6 +707,28 @@ mod tests { assert_eq!(derive_confirmation_timeout(5, 3), Duration::from_secs(36)); } + #[test] + fn suffix_watch_hash_skips_only_non_head_with_a_stored_hash() { + let hash = TxHash::repeat_byte(0xab); + let with_hash = InFlightTx { + fees: crate::l1::eip1559::Eip1559Fees { + base_fee_per_gas: 1, + max_priority_fee_per_gas: 1, + max_fee_per_gas: 2, + }, + tx_hash: Some(hash), + }; + let fees_only = InFlightTx { + fees: with_hash.fees, + tx_hash: None, + }; + + assert_eq!(suffix_watch_hash(10, 10, Some(with_hash)), None); + assert_eq!(suffix_watch_hash(10, 11, Some(with_hash)), Some(hash)); + assert_eq!(suffix_watch_hash(10, 11, Some(fees_only)), None); + assert_eq!(suffix_watch_hash(10, 11, None), None); + } + fn poster_config(anvil: &alloy::node_bindings::AnvilInstance) -> BatchPosterConfig { BatchPosterConfig { l1_submit_address: alloy_primitives::Address::repeat_byte(0x11), diff --git a/sequencer/src/recovery/flusher.rs b/sequencer/src/recovery/flusher.rs index e6d925ac..df7cd534 100644 --- a/sequencer/src/recovery/flusher.rs +++ b/sequencer/src/recovery/flusher.rs @@ -246,9 +246,18 @@ impl MempoolFlusher { .map_err(|e| FlushError::Provider(e.to_string()))?; // Bump the absolute estimate so no-ops can compete with pending batch - // txs at the same wallet nonces (shared ≥10% replacement rule). Safety + // txs at the same wallet nonces (shared ≥10% replacement rule). Both + // components grow equally; the helper also clamps tip ≤ cap so a + // near-zero-base estimate cannot produce ErrTipAboveFeeCap. Safety // does not depend on the no-op winning — `flush_and_wait` only returns // once Pending ≤ Safe. + // + // Residual gap: this is a one-shot bump of a *fresh* estimate, not of + // the pending tx's fees, so a replacement can still be underpriced + // when the two EIP-1559 components have moved independently. A + // rejected no-op hard-errors `flush_and_wait`; the orchestrator + // respawn retries. Tightening that needs the pending tx's fees (or a + // raise-on-underpriced-error loop), not a bigger one-shot multiplier. let (bumped_max_fee, bumped_priority_fee) = bumped_replacement_fees(fees.max_fee_per_gas, fees.max_priority_fee_per_gas); @@ -362,7 +371,8 @@ mod tests { // ── H5: replacement-fee bump keeps no-ops competitive ───────── // Rule itself lives in `l1::eip1559` (shared with the poster); the // flusher's use site is the `bumped_replacement_fees(...)` call in - // `submit_noops`. + // `submit_noops`. Equal ×1.1 growth plus the tip≤cap clamp are what + // keep a one-shot bump of a near-zero-base estimate valid. #[test] fn send_failure_error_summarizes_failed_slots() { From b09dbc1c513b8c3d947c497d5d3b2306e6572517 Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:34:14 +0800 Subject: [PATCH 5/9] fix(l1): raise poster floor on underpriced replacement errors When a same-nonce send is rejected as replacement-underpriced, immediately raise the stored head-nonce floor from the attempted fees so the next tick self-corrects instead of repeating the same underpriced pair. --- sequencer/src/l1/submitter/poster.rs | 60 +++++++++++++++++++++++++--- 1 file changed, 55 insertions(+), 5 deletions(-) diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index 1b68fe89..2ac4db10 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -94,8 +94,9 @@ pub struct EthereumBatchPoster { /// /// Process-local, so the floor is best-effort, not an invariant. A restart /// (or a send whose response is lost after the node accepted) re-opens the - /// underpriced-retry window for a cycle. A future hardening is to also - /// raise the floor on a "replacement transaction underpriced" send error. + /// underpriced-retry window for a cycle. A rejected "replacement transaction + /// underpriced" still raises the stored floor so the next tick self-corrects + /// without waiting for a confirmation timeout. in_flight: Arc>>, /// Test-only: next `send_batch_at_nonce` returns Err without broadcasting, /// so callers can assert the in-flight map is not updated on send failure. @@ -253,6 +254,11 @@ fn suffix_watch_hash(head_nonce: u64, nonce: u64, existing: Option) existing.and_then(|tx| tx.tx_hash) } +/// geth rejects same-nonce replacements below the ≥10% bump threshold. +fn is_replacement_underpriced(err: &str) -> bool { + err.contains("replacement transaction underpriced") +} + fn derive_confirmation_timeout( confirmation_depth: u64, seconds_per_block: u64, @@ -334,9 +340,28 @@ impl BatchPoster for EthereumBatchPoster { None }; let fees = fees_for_nonce(estimate, prior); - let pending = self.send_batch_at_nonce(payload, next_nonce, &fees).await?; + let pending = match self.send_batch_at_nonce(payload, next_nonce, &fees).await { + Ok(pending) => pending, + Err(BatchPosterError::Provider(ref msg)) if is_replacement_underpriced(msg) => { + // Node rejected the replacement fee — raise the floor from + // what we just tried so the next tick clears the threshold + // without waiting for a confirmation timeout. + let raised = fees_for_nonce(fees, Some(fees)); + self.in_flight.lock().expect("in_flight lock").insert( + next_nonce, + InFlightTx { + fees: raised, + tx_hash: existing.and_then(|tx| tx.tx_hash), + }, + ); + return Err(BatchPosterError::Provider(msg.clone())); + } + Err(err) => return Err(err), + }; // Record only after a successful broadcast — a failed send must - // not raise the replacement floor for the next tick. + // not raise the replacement floor for the next tick (except the + // underpriced path above, which self-corrects against a live pending + // tx the node already holds). let tx_hash = *pending.tx_hash(); self.in_flight.lock().expect("in_flight lock").insert( next_nonce, @@ -506,7 +531,8 @@ mod tests { use super::{ BatchPoster, BatchPosterConfig, BatchPosterError, EthereumBatchPoster, InFlightTx, TxHash, - derive_confirmation_timeout, mock::MockBatchPoster, suffix_watch_hash, + derive_confirmation_timeout, is_replacement_underpriced, mock::MockBatchPoster, + suffix_watch_hash, }; use crate::l1::watermark::WalletNonceWatermarkSink; use alloy::node_bindings::Anvil; @@ -729,6 +755,30 @@ mod tests { assert_eq!(suffix_watch_hash(10, 11, None), None); } + #[test] + fn is_replacement_underpriced_matches_geth_message() { + assert!(is_replacement_underpriced( + "server returned an error response: error code -32000: replacement transaction underpriced" + )); + assert!(!is_replacement_underpriced("nonce too low")); + assert!(!is_replacement_underpriced( + "max priority fee per gas higher than max fee per gas" + )); + } + + #[test] + fn underpriced_send_raises_floor_from_attempted_fees() { + let attempted = crate::l1::eip1559::Eip1559Fees { + base_fee_per_gas: 20_000_000_000, + max_priority_fee_per_gas: 1_000_000_000, + max_fee_per_gas: 41_000_000_000, + }; + let raised = crate::l1::eip1559::fees_for_nonce(attempted, Some(attempted)); + assert!(raised.max_fee_per_gas > attempted.max_fee_per_gas); + assert!(raised.max_priority_fee_per_gas > attempted.max_priority_fee_per_gas); + assert!(raised.max_priority_fee_per_gas <= raised.max_fee_per_gas); + } + fn poster_config(anvil: &alloy::node_bindings::AnvilInstance) -> BatchPosterConfig { BatchPosterConfig { l1_submit_address: alloy_primitives::Address::repeat_byte(0x11), From b60f60830b5b1ecbeece78e376840aa73ddb57d4 Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:10:01 +0800 Subject: [PATCH 6/9] fix(l1): estimate gas without pending nonce on poster replacements Anvil rejects eth_estimateGas with "nonce too low" when the same nonce is already pending, so replacements never reached eth_sendRawTransaction. --- sequencer/src/l1/submitter/poster.rs | 156 ++++++++++++++++++++++++++- 1 file changed, 151 insertions(+), 5 deletions(-) diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index 2ac4db10..9224c7c3 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -171,6 +171,8 @@ impl EthereumBatchPoster { payload: Vec, nonce: u64, fees: &Eip1559Fees, + // True when this nonce already has a pending broadcast we are replacing. + replace_pending: bool, ) -> Result, BatchPosterError> { #[cfg(test)] { @@ -181,12 +183,28 @@ impl EthereumBatchPoster { } } let input_box = InputBox::new(self.config.l1_submit_address, &self.provider); - input_box + let call = input_box .addInput(self.config.app_address, payload.into()) - .nonce(nonce) .max_fee_per_gas(fees.max_fee_per_gas) - .max_priority_fee_per_gas(fees.max_priority_fee_per_gas) - .send() + .max_priority_fee_per_gas(fees.max_priority_fee_per_gas); + + // Same-nonce replacement: estimate gas *without* the pending nonce. + // Anvil (and geth's pending simulation) apply mempool nonce policy to + // `eth_estimateGas` and reject with "nonce too low" when that nonce is + // already pending — so the filler's estimate-with-nonce never reaches + // `eth_sendRawTransaction`. Pin the gas limit first so the filler + // skips a second estimate that would include the nonce. + let call = if replace_pending { + let gas = call + .estimate_gas() + .await + .map_err(|err| BatchPosterError::Provider(err.to_string()))?; + call.gas(gas).nonce(nonce) + } else { + call.nonce(nonce) + }; + + call.send() .await .map_err(|err| BatchPosterError::Provider(err.to_string())) } @@ -340,7 +358,10 @@ impl BatchPoster for EthereumBatchPoster { None }; let fees = fees_for_nonce(estimate, prior); - let pending = match self.send_batch_at_nonce(payload, next_nonce, &fees).await { + let pending = match self + .send_batch_at_nonce(payload, next_nonce, &fees, prior.is_some()) + .await + { Ok(pending) => pending, Err(BatchPosterError::Provider(ref msg)) if is_replacement_underpriced(msg) => { // Node rejected the replacement fee — raise the floor from @@ -849,6 +870,131 @@ mod tests { ); } + /// Same-nonce replacement must reach `eth_sendRawTransaction` while the + /// original is still pending. Without pinning gas from a nonce-free + /// estimate, Anvil rejects `eth_estimateGas(..., nonce=N, block=pending)` + /// with "nonce too low" and the replacement never broadcasts. + #[tokio::test] + async fn submit_batches_replaces_pending_tx_after_confirmation_timeout() { + require_anvil(); + let anvil = Anvil::default().arg("--no-mining").timeout(30_000).spawn(); + let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + let submitter = alloy_primitives::address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); + let provider = crate::l1::provider::create_signer_provider(&anvil.endpoint(), key, false) + .expect("signer provider"); + let poster = EthereumBatchPoster::new(provider.clone(), poster_config(&anvil)); + let sink = RecordingWatermarkSink::passing(); + + let base_nonce = provider + .get_transaction_count(submitter) + .await + .expect("base nonce"); + + let first_hashes = poster + .submit_batches(vec![vec![0u8; 4]], &sink) + .await + .expect("first submit parks a pending tx"); + assert_eq!(first_hashes.len(), 1); + let first_hash = first_hashes[0]; + + let pending_after_first = provider + .get_transaction_count(submitter) + .block_id(BlockNumberOrTag::Pending.into()) + .await + .expect("pending nonce"); + let latest_after_first = provider + .get_transaction_count(submitter) + .block_id(BlockNumberOrTag::Latest.into()) + .await + .expect("latest nonce"); + assert_eq!( + pending_after_first, + base_nonce + 1, + "first send must occupy the mempool slot" + ); + assert_eq!( + latest_after_first, base_nonce, + "mining is disabled; latest must not advance" + ); + + let prior_fees = poster + .in_flight_fees_for_test() + .get(&base_nonce) + .copied() + .expect("first send records in-flight fees"); + + // Confirmation watch timed out inside the first submit; the next tick + // must bump fees and broadcast a same-nonce replacement. + let second_hashes = poster + .submit_batches(vec![vec![0u8; 4]], &sink) + .await + .expect("replacement must clear gas estimation and broadcast"); + assert_eq!(second_hashes.len(), 1); + let replacement_hash = second_hashes[0]; + assert_ne!( + replacement_hash, first_hash, + "replacement must be a distinct tx hash" + ); + + let sent = poster + .in_flight_fees_for_test() + .get(&base_nonce) + .copied() + .expect("replacement records bumped fees"); + let (bumped_max, bumped_prio) = crate::l1::eip1559::bumped_replacement_fees( + prior_fees.max_fee_per_gas, + prior_fees.max_priority_fee_per_gas, + ); + assert!( + sent.max_fee_per_gas >= bumped_max, + "replacement max_fee must clear floor: sent={} floor={bumped_max}", + sent.max_fee_per_gas + ); + assert!( + sent.max_priority_fee_per_gas >= bumped_prio, + "replacement priority must clear floor: sent={} floor={bumped_prio}", + sent.max_priority_fee_per_gas + ); + + // Pending still one slot ahead of latest until we mine. + let pending_after_replace = provider + .get_transaction_count(submitter) + .block_id(BlockNumberOrTag::Pending.into()) + .await + .expect("pending after replace"); + assert_eq!( + pending_after_replace, + base_nonce + 1, + "replacement keeps a single pending nonce slot" + ); + + let _: serde_json::Value = provider + .raw_request("evm_mine".into(), ()) + .await + .expect("mine replacement"); + + let latest_after_mine = provider + .get_transaction_count(submitter) + .block_id(BlockNumberOrTag::Latest.into()) + .await + .expect("latest after mine"); + assert_eq!( + latest_after_mine, + base_nonce + 1, + "mined replacement must advance the account nonce" + ); + + let receipt = provider + .get_transaction_receipt(replacement_hash) + .await + .expect("receipt rpc") + .expect("replacement must have a receipt after mining"); + assert_eq!( + receipt.transaction_hash, replacement_hash, + "mined receipt must belong to the replacement tx" + ); + } + /// When Latest advances past a nonce, that nonce's fee floor is dropped so a /// later tip send is not incorrectly floored by stale in-flight state. #[tokio::test] From 9256c40e7ac8927efe7daa9f605aed6ffe3cf272 Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Wed, 26 Aug 2026 14:10:01 +0800 Subject: [PATCH 7/9] test(l1): harden poster same-nonce replacement E2E assertions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Assert the original pending tx is evicted and on-wire fees clear the ≥10% floor after mining resumes. --- sequencer/src/l1/submitter/poster.rs | 49 ++++++++++++++++++++++++---- 1 file changed, 42 insertions(+), 7 deletions(-) diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index 9224c7c3..73259554 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -870,10 +870,15 @@ mod tests { ); } - /// Same-nonce replacement must reach `eth_sendRawTransaction` while the - /// original is still pending. Without pinning gas from a nonce-free - /// estimate, Anvil rejects `eth_estimateGas(..., nonce=N, block=pending)` - /// with "nonce too low" and the replacement never broadcasts. + /// Review E2E (jplgarcia): Anvil `--no-mining` → original poster tx → + /// confirmation timeout → real same-nonce replacement → resume mining → + /// assert receipt / nonce progression. + /// + /// Pins the gas-estimation hole that blocked the replacement path: with the + /// original still pending, Anvil rejects `eth_estimateGas(..., nonce=N, + /// block=pending)` as "nonce too low", so the filler never reaches + /// `eth_sendRawTransaction`. The poster must estimate without that nonce + /// and pin gas on the replacement. #[tokio::test] async fn submit_batches_replaces_pending_tx_after_confirmation_timeout() { require_anvil(); @@ -890,6 +895,7 @@ mod tests { .await .expect("base nonce"); + // 1) Original poster tx parks in the mempool (mining disabled). let first_hashes = poster .submit_batches(vec![vec![0u8; 4]], &sink) .await @@ -923,8 +929,8 @@ mod tests { .copied() .expect("first send records in-flight fees"); - // Confirmation watch timed out inside the first submit; the next tick - // must bump fees and broadcast a same-nonce replacement. + // 2) Confirmation watch timed out; next tick must bump fees and + // broadcast a same-nonce replacement (the gas-estimate fix). let second_hashes = poster .submit_batches(vec![vec![0u8; 4]], &sink) .await @@ -956,7 +962,7 @@ mod tests { sent.max_priority_fee_per_gas ); - // Pending still one slot ahead of latest until we mine. + // Still one pending slot until mining resumes. let pending_after_replace = provider .get_transaction_count(submitter) .block_id(BlockNumberOrTag::Pending.into()) @@ -968,6 +974,7 @@ mod tests { "replacement keeps a single pending nonce slot" ); + // 3) Resume mining and assert receipt / nonce progression. let _: serde_json::Value = provider .raw_request("evm_mine".into(), ()) .await @@ -993,6 +1000,34 @@ mod tests { receipt.transaction_hash, replacement_hash, "mined receipt must belong to the replacement tx" ); + assert!( + provider + .get_transaction_receipt(first_hash) + .await + .expect("original receipt rpc") + .is_none(), + "original pending tx must be evicted by the replacement" + ); + + // On-wire fees must clear the ≥10% floor (not just the in-memory record). + use alloy::consensus::Transaction as _; + let mined = provider + .get_transaction_by_hash(replacement_hash) + .await + .expect("get replacement tx") + .expect("replacement tx must be fetchable after mining"); + assert!( + mined.max_fee_per_gas() >= bumped_max, + "mined max_fee must clear floor: on_wire={} floor={bumped_max}", + mined.max_fee_per_gas() + ); + let on_wire_prio = mined + .max_priority_fee_per_gas() + .expect("replacement must be EIP-1559"); + assert!( + on_wire_prio >= bumped_prio, + "mined priority must clear floor: on_wire={on_wire_prio} floor={bumped_prio}" + ); } /// When Latest advances past a nonce, that nonce's fee floor is dropped so a From c67b384cd0d2a6292a1fcbc3764d91c28734f7d2 Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Thu, 27 Aug 2026 22:23:29 +0800 Subject: [PATCH 8/9] fix(l1): rebroadcast suffix floors and always pad gas estimates Drop head-only skip-and-watch so every pending nonce is re-sent and floored. Estimate gas without a nonce on every send (and in the flusher), pin a +10% pad, and match underpriced errors case-insensitively. --- sequencer/src/l1/submitter/poster.rs | 386 +++++++++++++++++---------- sequencer/src/recovery/flusher.rs | 27 +- 2 files changed, 268 insertions(+), 145 deletions(-) diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index 73259554..783b7a40 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -16,20 +16,15 @@ use crate::l1::eip1559::{Eip1559Fees, estimate_fees, fees_for_nonce}; use crate::l1::partition::{decode_evm_advance_input, get_input_added_events_ordered}; use crate::l1::watermark::WalletNonceWatermarkSink; use std::collections::BTreeMap; -#[cfg(test)] -use std::sync::atomic::{AtomicBool, Ordering}; use std::sync::{Arc, Mutex}; pub type TxHash = alloy_primitives::B256; -/// Last successful broadcast at a wallet nonce: fees we actually sent, and -/// the hash the next tick can keep watching if this nonce is no longer the -/// blocking head (so we do not replace it). `tx_hash` is `None` only in tests -/// that seed a fee floor without a prior send. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -struct InFlightTx { - fees: Eip1559Fees, - tx_hash: Option, +/// Pad an `eth_estimateGas` result so a tight estimate cannot mine as an +/// out-of-gas revert (which would burn the wallet-nonce slot with no +/// `InputAdded` and desynchronize payload↔nonce assignment). +fn pad_gas_estimate(gas: u64) -> u64 { + gas.saturating_add(gas / 10) } #[derive(Debug, Clone)] @@ -82,26 +77,26 @@ pub trait BatchPoster: Send + Sync { pub struct EthereumBatchPoster { provider: DynProvider, config: BatchPosterConfig, - /// Fees + tx hash of the last successful broadcast per wallet nonce still - /// ≥ Latest. + /// Fees of the last successful broadcast (or underpriced raise) per wallet + /// nonce still ≥ Latest. /// - /// Same-nonce retries of the **head** (Latest) nonce floor a fresh - /// estimate against [`crate::l1::eip1559::bumped_replacement_fees`] of - /// this record so a flat market cannot re-broadcast underpriced - /// replacements. Suffix nonces already in the map are left in the mempool: - /// only the head can be blocking, and re-escalating the whole unconfirmed - /// suffix compounds fees for txs that cannot mine until the head does. + /// Same-nonce retries floor a fresh estimate against + /// [`crate::l1::eip1559::bumped_replacement_fees`] of this record for + /// **every** pending nonce (at-least-once re-broadcast of the whole + /// unconfirmed suffix). Overpay is ~×1.1 per suffix tx per rare timeout + /// round — cheaper than silently dropping a batch or watching a hash the + /// node no longer holds after failover/restart/eviction. /// /// Process-local, so the floor is best-effort, not an invariant. A restart /// (or a send whose response is lost after the node accepted) re-opens the /// underpriced-retry window for a cycle. A rejected "replacement transaction /// underpriced" still raises the stored floor so the next tick self-corrects /// without waiting for a confirmation timeout. - in_flight: Arc>>, - /// Test-only: next `send_batch_at_nonce` returns Err without broadcasting, - /// so callers can assert the in-flight map is not updated on send failure. + in_flight: Arc>>, + /// Test-only: next `send_batch_at_nonce` returns this error string without + /// broadcasting (so callers can assert map updates / underpriced handling). #[cfg(test)] - fail_next_send: Arc, + fail_next_send: Arc>>, } impl EthereumBatchPoster { @@ -111,41 +106,29 @@ impl EthereumBatchPoster { config, in_flight: Arc::new(Mutex::new(BTreeMap::new())), #[cfg(test)] - fail_next_send: Arc::new(AtomicBool::new(false)), + fail_next_send: Arc::new(Mutex::new(None)), } } #[cfg(test)] pub(crate) fn in_flight_fees_for_test(&self) -> BTreeMap { - self.in_flight - .lock() - .expect("in_flight lock") - .iter() - .map(|(&nonce, tx)| (nonce, tx.fees)) - .collect() + self.in_flight.lock().expect("in_flight lock").clone() } #[cfg(test)] pub(crate) fn seed_in_flight_fees_for_test(&self, fees: BTreeMap) { - let mut in_flight = self.in_flight.lock().expect("in_flight lock"); - in_flight.clear(); - for (nonce, fees) in fees { - in_flight.insert( - nonce, - InFlightTx { - fees, - // Tests that seed a floor then submit are replacing the - // head nonce; the hash is only used to skip suffix - // re-broadcast, which those tests do not exercise. - tx_hash: None, - }, - ); - } + *self.in_flight.lock().expect("in_flight lock") = fees; } #[cfg(test)] pub(crate) fn fail_next_send_for_test(&self) { - self.fail_next_send.store(true, Ordering::SeqCst); + *self.fail_next_send.lock().expect("fail_next_send lock") = + Some("test-injected send failure".to_string()); + } + + #[cfg(test)] + pub(crate) fn fail_next_send_with_for_test(&self, message: &str) { + *self.fail_next_send.lock().expect("fail_next_send lock") = Some(message.to_string()); } /// Conservative upper-bound timeout for waiting on confirmations, derived @@ -171,15 +154,16 @@ impl EthereumBatchPoster { payload: Vec, nonce: u64, fees: &Eip1559Fees, - // True when this nonce already has a pending broadcast we are replacing. - replace_pending: bool, ) -> Result, BatchPosterError> { #[cfg(test)] { - if self.fail_next_send.swap(false, Ordering::SeqCst) { - return Err(BatchPosterError::Provider( - "test-injected send failure".to_string(), - )); + if let Some(message) = self + .fail_next_send + .lock() + .expect("fail_next_send lock") + .take() + { + return Err(BatchPosterError::Provider(message)); } } let input_box = InputBox::new(self.config.l1_submit_address, &self.provider); @@ -188,23 +172,26 @@ impl EthereumBatchPoster { .max_fee_per_gas(fees.max_fee_per_gas) .max_priority_fee_per_gas(fees.max_priority_fee_per_gas); - // Same-nonce replacement: estimate gas *without* the pending nonce. - // Anvil (and geth's pending simulation) apply mempool nonce policy to - // `eth_estimateGas` and reject with "nonce too low" when that nonce is - // already pending — so the filler's estimate-with-nonce never reaches - // `eth_sendRawTransaction`. Pin the gas limit first so the filler - // skips a second estimate that would include the nonce. - let call = if replace_pending { - let gas = call - .estimate_gas() + // Always estimate without an explicit nonce and pin gas (+10% pad) + // before send. Anvil applies mempool nonce policy to pending + // `eth_estimateGas` and rejects with "nonce too low" when that nonce + // is already pending — including the restart shape where we have no + // in-flight floor yet. (geth typically skips nonce checks in + // estimateGas via SkipNonceChecks; the Anvil path is what bites + // locally/CI.) Explicit `CallBuilder::estimate_gas` also pins + // block=Latest (filler default is pending), either of which avoids + // Anvil's check. With gas + both fee fields set, the GasFiller is + // Finished — this estimate replaces the filler's rather than adding + // a second round-trip. + let gas = pad_gas_estimate( + call.estimate_gas() .await - .map_err(|err| BatchPosterError::Provider(err.to_string()))?; - call.gas(gas).nonce(nonce) - } else { - call.nonce(nonce) - }; + .map_err(|err| BatchPosterError::Provider(err.to_string()))?, + ); - call.send() + call.gas(gas) + .nonce(nonce) + .send() .await .map_err(|err| BatchPosterError::Provider(err.to_string())) } @@ -223,11 +210,10 @@ impl EthereumBatchPoster { /// /// Timeouts return `Ok(())` rather than `Err` because the safe response is /// "re-enter `submit_batches` on the next tick" — which re-estimates fees, - /// floors the **head** nonce to an explicit ≥10% replacement bump against - /// any still in-flight same-nonce submission, leaves already-broadcast - /// suffix txs in the mempool, and re-submits only what still needs a - /// replacement. The wallet-nonce ordering invariant above guarantees we - /// cannot accidentally skip work by returning early here. + /// floors **every** still-pending nonce against its in-flight record, and + /// at-least-once re-broadcasts the whole unconfirmed suffix. The + /// wallet-nonce ordering invariant above guarantees we cannot accidentally + /// skip work by returning early here. async fn wait_for_confirmations(&self, tx_hashes: &[TxHash]) -> Result<(), BatchPosterError> { let timeout = self.confirmation_timeout(); for tx_hash in tx_hashes { @@ -263,18 +249,13 @@ impl EthereumBatchPoster { } } -/// If this nonce is behind the blocking head and already in the mempool, keep -/// watching the original hash instead of replacing it. -fn suffix_watch_hash(head_nonce: u64, nonce: u64, existing: Option) -> Option { - if nonce == head_nonce { - return None; - } - existing.and_then(|tx| tx.tx_hash) -} - -/// geth rejects same-nonce replacements below the ≥10% bump threshold. +/// geth-family nodes reject same-nonce replacements below the ≥10% bump +/// threshold. Haystack is lowercased so Besu's capitalized wording still +/// matches; Nethermind/Erigon reword the error and are not covered — a miss +/// during bootstrap stalls rather than degrades. fn is_replacement_underpriced(err: &str) -> bool { - err.contains("replacement transaction underpriced") + err.to_ascii_lowercase() + .contains("replacement transaction underpriced") } fn derive_confirmation_timeout( @@ -338,43 +319,24 @@ impl BatchPoster for EthereumBatchPoster { .map_err(BatchPosterError::Provider)?; let mut tx_hashes = Vec::with_capacity(payloads.len()); - let head_nonce = next_nonce; for payload in payloads { - let existing = { + let prior = { let in_flight = self.in_flight.lock().expect("in_flight lock"); in_flight.get(&next_nonce).copied() }; - - if let Some(tx_hash) = suffix_watch_hash(head_nonce, next_nonce, existing) { - tx_hashes.push(tx_hash); - next_nonce = next_nonce.saturating_add(1); - continue; - } - - let prior = if next_nonce == head_nonce { - existing.map(|tx| tx.fees) - } else { - None - }; let fees = fees_for_nonce(estimate, prior); - let pending = match self - .send_batch_at_nonce(payload, next_nonce, &fees, prior.is_some()) - .await - { + let pending = match self.send_batch_at_nonce(payload, next_nonce, &fees).await { Ok(pending) => pending, Err(BatchPosterError::Provider(ref msg)) if is_replacement_underpriced(msg) => { // Node rejected the replacement fee — raise the floor from // what we just tried so the next tick clears the threshold // without waiting for a confirmation timeout. let raised = fees_for_nonce(fees, Some(fees)); - self.in_flight.lock().expect("in_flight lock").insert( - next_nonce, - InFlightTx { - fees: raised, - tx_hash: existing.and_then(|tx| tx.tx_hash), - }, - ); + self.in_flight + .lock() + .expect("in_flight lock") + .insert(next_nonce, raised); return Err(BatchPosterError::Provider(msg.clone())); } Err(err) => return Err(err), @@ -384,13 +346,10 @@ impl BatchPoster for EthereumBatchPoster { // underpriced path above, which self-corrects against a live pending // tx the node already holds). let tx_hash = *pending.tx_hash(); - self.in_flight.lock().expect("in_flight lock").insert( - next_nonce, - InFlightTx { - fees, - tx_hash: Some(tx_hash), - }, - ); + self.in_flight + .lock() + .expect("in_flight lock") + .insert(next_nonce, fees); debug!( tx_nonce = next_nonce, %tx_hash, @@ -551,9 +510,9 @@ mod tests { use std::time::Duration; use super::{ - BatchPoster, BatchPosterConfig, BatchPosterError, EthereumBatchPoster, InFlightTx, TxHash, + BatchPoster, BatchPosterConfig, BatchPosterError, EthereumBatchPoster, derive_confirmation_timeout, is_replacement_underpriced, mock::MockBatchPoster, - suffix_watch_hash, + pad_gas_estimate, }; use crate::l1::watermark::WalletNonceWatermarkSink; use alloy::node_bindings::Anvil; @@ -755,32 +714,20 @@ mod tests { } #[test] - fn suffix_watch_hash_skips_only_non_head_with_a_stored_hash() { - let hash = TxHash::repeat_byte(0xab); - let with_hash = InFlightTx { - fees: crate::l1::eip1559::Eip1559Fees { - base_fee_per_gas: 1, - max_priority_fee_per_gas: 1, - max_fee_per_gas: 2, - }, - tx_hash: Some(hash), - }; - let fees_only = InFlightTx { - fees: with_hash.fees, - tx_hash: None, - }; - - assert_eq!(suffix_watch_hash(10, 10, Some(with_hash)), None); - assert_eq!(suffix_watch_hash(10, 11, Some(with_hash)), Some(hash)); - assert_eq!(suffix_watch_hash(10, 11, Some(fees_only)), None); - assert_eq!(suffix_watch_hash(10, 11, None), None); + fn pad_gas_estimate_adds_ten_percent() { + assert_eq!(pad_gas_estimate(100_000), 110_000); + assert_eq!(pad_gas_estimate(0), 0); + assert_eq!(pad_gas_estimate(u64::MAX), u64::MAX); } #[test] - fn is_replacement_underpriced_matches_geth_message() { + fn is_replacement_underpriced_matches_geth_family_case_insensitively() { assert!(is_replacement_underpriced( "server returned an error response: error code -32000: replacement transaction underpriced" )); + assert!(is_replacement_underpriced( + "Replacement transaction underpriced" // Besu capitalizes + )); assert!(!is_replacement_underpriced("nonce too low")); assert!(!is_replacement_underpriced( "max priority fee per gas higher than max fee per gas" @@ -877,8 +824,8 @@ mod tests { /// Pins the gas-estimation hole that blocked the replacement path: with the /// original still pending, Anvil rejects `eth_estimateGas(..., nonce=N, /// block=pending)` as "nonce too low", so the filler never reaches - /// `eth_sendRawTransaction`. The poster must estimate without that nonce - /// and pin gas on the replacement. + /// `eth_sendRawTransaction`. The poster always estimates without that nonce + /// (Latest) and pins padded gas on every send. #[tokio::test] async fn submit_batches_replaces_pending_tx_after_confirmation_timeout() { require_anvil(); @@ -1121,4 +1068,169 @@ mod tests { "failed send must leave the prior in-flight floor untouched" ); } + + /// Underpriced rejection raises the floor from the fees we attempted, so the + /// next tick clears geth's ≥10% threshold without waiting for timeout. + #[tokio::test] + async fn submit_batches_underpriced_raises_floor_from_attempted_fees() { + require_anvil(); + let anvil = Anvil::default().arg("--no-mining").timeout(30_000).spawn(); + let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + let submitter = alloy_primitives::address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); + let provider = crate::l1::provider::create_signer_provider(&anvil.endpoint(), key, false) + .expect("signer provider"); + let poster = EthereumBatchPoster::new(provider.clone(), poster_config(&anvil)); + let sink = RecordingWatermarkSink::passing(); + + let base_nonce = provider + .get_transaction_count(submitter) + .await + .expect("base nonce"); + let prior = crate::l1::eip1559::Eip1559Fees { + base_fee_per_gas: 1, + max_priority_fee_per_gas: 50_000_000, + max_fee_per_gas: 100_000_000_000, + }; + poster.seed_in_flight_fees_for_test(BTreeMap::from([(base_nonce, prior)])); + poster.fail_next_send_with_for_test( + "server returned an error response: error code -32000: Replacement transaction underpriced", + ); + + let result = poster.submit_batches(vec![vec![0u8; 4]], &sink).await; + assert!( + matches!( + result, + Err(BatchPosterError::Provider(ref msg)) + if is_replacement_underpriced(msg) + ), + "underpriced injection must surface, got {result:?}" + ); + + let raised = poster + .in_flight_fees_for_test() + .get(&base_nonce) + .copied() + .expect("underpriced path must raise the floor"); + // Attempted fees clear prior; raised clears attempted. + let (floor_max, floor_prio) = crate::l1::eip1559::bumped_replacement_fees( + prior.max_fee_per_gas, + prior.max_priority_fee_per_gas, + ); + assert!(raised.max_fee_per_gas > floor_max); + assert!(raised.max_priority_fee_per_gas > floor_prio); + assert!(raised.max_priority_fee_per_gas <= raised.max_fee_per_gas); + } + + /// Restart shape: empty in-flight map vs a live pending tx. Unconditional + /// nonce-free gas estimate must still reach `eth_sendRawTransaction`. + #[tokio::test] + async fn submit_batches_replaces_pending_without_in_flight_floor() { + require_anvil(); + let anvil = Anvil::default().arg("--no-mining").timeout(30_000).spawn(); + let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + let provider = crate::l1::provider::create_signer_provider(&anvil.endpoint(), key, false) + .expect("signer provider"); + let poster = EthereumBatchPoster::new(provider.clone(), poster_config(&anvil)); + let sink = RecordingWatermarkSink::passing(); + + let first_hashes = poster + .submit_batches(vec![vec![0u8; 4]], &sink) + .await + .expect("first submit parks a pending tx"); + let first_hash = first_hashes[0]; + + // Simulate restart / lost response: forget the floor while the tx is + // still pending on the node. + poster.seed_in_flight_fees_for_test(BTreeMap::new()); + + let result = poster.submit_batches(vec![vec![0u8; 4]], &sink).await; + match result { + Ok(second_hashes) => { + assert_ne!( + second_hashes[0], first_hash, + "if fees moved, the untracked replacement must be a new hash" + ); + } + Err(BatchPosterError::Provider(msg)) => { + let lower = msg.to_ascii_lowercase(); + assert!( + !lower.contains("nonce too low"), + "untracked replacement must not die at estimateGas: {msg}" + ); + // Without a floor the re-estimate can match the pending tx + // exactly ("already imported") or land underpriced — either + // proves we reached send, which is the restart-shape hole. + assert!( + lower.contains("already imported") || is_replacement_underpriced(&msg), + "unexpected send failure for untracked replacement: {msg}" + ); + } + Err(other) => panic!("unexpected error: {other:?}"), + } + } + + /// Multi-nonce suffix: every pending nonce is re-broadcast and floored on + /// retry (no skip-and-watch). Three payloads → three replacements. + #[tokio::test] + async fn submit_batches_rebroadcasts_and_floors_entire_pending_suffix() { + require_anvil(); + let anvil = Anvil::default().arg("--no-mining").timeout(30_000).spawn(); + let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + let submitter = alloy_primitives::address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); + let provider = crate::l1::provider::create_signer_provider(&anvil.endpoint(), key, false) + .expect("signer provider"); + let poster = EthereumBatchPoster::new(provider.clone(), poster_config(&anvil)); + let sink = RecordingWatermarkSink::passing(); + + let base_nonce = provider + .get_transaction_count(submitter) + .await + .expect("base nonce"); + let payloads = vec![vec![0u8; 4], vec![1u8; 4], vec![2u8; 4]]; + + let first_hashes = poster + .submit_batches(payloads.clone(), &sink) + .await + .expect("first multi-nonce submit"); + assert_eq!(first_hashes.len(), 3); + let prior_fees = poster.in_flight_fees_for_test(); + assert_eq!(prior_fees.len(), 3); + + let second_hashes = poster + .submit_batches(payloads, &sink) + .await + .expect("suffix rebroadcast"); + assert_eq!(second_hashes.len(), 3); + for (first, second) in first_hashes.iter().zip(second_hashes.iter()) { + assert_ne!( + first, second, + "each nonce must be replaced, not skip-watched" + ); + } + + let sent = poster.in_flight_fees_for_test(); + for offset in 0..3u64 { + let nonce = base_nonce + offset; + let prior = prior_fees.get(&nonce).copied().expect("prior fees"); + let next = sent.get(&nonce).copied().expect("replacement fees"); + let (bumped_max, bumped_prio) = crate::l1::eip1559::bumped_replacement_fees( + prior.max_fee_per_gas, + prior.max_priority_fee_per_gas, + ); + assert!(next.max_fee_per_gas >= bumped_max); + assert!(next.max_priority_fee_per_gas >= bumped_prio); + } + + let _: serde_json::Value = provider + .raw_request("evm_mine".into(), ()) + .await + .expect("mine"); + // One block includes the contiguous suffix in nonce order. + let latest = provider + .get_transaction_count(submitter) + .block_id(BlockNumberOrTag::Latest.into()) + .await + .expect("latest"); + assert_eq!(latest, base_nonce + 3); + } } diff --git a/sequencer/src/recovery/flusher.rs b/sequencer/src/recovery/flusher.rs index df7cd534..f70944ff 100644 --- a/sequencer/src/recovery/flusher.rs +++ b/sequencer/src/recovery/flusher.rs @@ -246,11 +246,10 @@ impl MempoolFlusher { .map_err(|e| FlushError::Provider(e.to_string()))?; // Bump the absolute estimate so no-ops can compete with pending batch - // txs at the same wallet nonces (shared ≥10% replacement rule). Both - // components grow equally; the helper also clamps tip ≤ cap so a - // near-zero-base estimate cannot produce ErrTipAboveFeeCap. Safety - // does not depend on the no-op winning — `flush_and_wait` only returns - // once Pending ≤ Safe. + // txs at the same wallet nonces (shared ≥10% replacement helper — + // symmetric ×1.1+1 on both EIP-1559 components). Safety does not + // depend on the no-op winning — `flush_and_wait` only returns once + // Pending ≤ Safe. // // Residual gap: this is a one-shot bump of a *fresh* estimate, not of // the pending tx's fees, so a replacement can still be underpriced @@ -273,17 +272,29 @@ impl MempoolFlusher { let mut tx_hashes = Vec::new(); let mut send_failures = Vec::new(); for nonce in from_nonce..to_nonce { - let tx = alloy::rpc::types::TransactionRequest::default() + // Estimate without the pending nonce and pin gas (+10% pad) — + // same Anvil "nonce too low" hole as the batch poster. With gas + // + both fee fields set, the GasFiller skips a second estimate. + let tx_base = alloy::rpc::types::TransactionRequest::default() .with_to(self.address) .with_value(U256::ZERO) - .with_nonce(nonce) .with_max_fee_per_gas(bumped_max_fee) .with_max_priority_fee_per_gas(bumped_priority_fee); + let gas = match self.provider.estimate_gas(tx_base.clone()).await { + Ok(gas) => gas.saturating_add(gas / 10), + Err(e) => { + let message = e.to_string(); + error!(nonce, error = %message, "flush no-op gas estimate failed"); + send_failures.push((nonce, message)); + continue; + } + }; + let tx = tx_base.with_gas_limit(gas).with_nonce(nonce); match self.provider.send_transaction(tx).await { Ok(pending) => { let tx_hash = *pending.tx_hash(); - debug!(nonce, %tx_hash, "flush no-op submitted"); + debug!(nonce, %tx_hash, gas, "flush no-op submitted"); tx_hashes.push(tx_hash); } Err(e) => { From b69c374894d18c59f1c8b97405be07f16aab65f7 Mon Sep 17 00:00:00 2001 From: Stephen Chen <20940639+stephenctw@users.noreply.github.com> Date: Sat, 5 Sep 2026 21:55:49 +0800 Subject: [PATCH 9/9] fix(l1): bound replacement fees with a suggestion-relative ceiling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Clamp max_fee to max(3×estimate, 2 gwei) inside fees_for_nonce, with a starved-tip escape valve and a Held tick that still probes the mempool without raising the floor. Match already-known / insufficient-funds, and price flusher no-ops at the same ceiling so recovery can clear anything the poster produces. Held sleeps on confirmation cadence — not an internal retry loop. --- sequencer/src/l1/eip1559.rs | 432 +++++++++++++----- sequencer/src/l1/submitter/config.rs | 9 + sequencer/src/l1/submitter/mod.rs | 5 +- sequencer/src/l1/submitter/poster.rs | 366 +++++++++++---- sequencer/src/l1/submitter/worker.rs | 71 ++- sequencer/src/recovery/flusher.rs | 75 ++- sequencer/src/runtime/workers.rs | 2 + .../tests/batch_submitter_integration.rs | 12 +- 8 files changed, 688 insertions(+), 284 deletions(-) diff --git a/sequencer/src/l1/eip1559.rs b/sequencer/src/l1/eip1559.rs index 147300b3..616299a4 100644 --- a/sequencer/src/l1/eip1559.rs +++ b/sequencer/src/l1/eip1559.rs @@ -5,11 +5,49 @@ //! //! Used by the batch poster (submission + retry), the fee oracle (charge), and //! the mempool flusher (no-op replacement). +//! +//! ## Fee ceiling +//! +//! Same-nonce retries floor against the last recorded fees for **every** +//! pending nonce. Without a bound, a long L1 stall (or a 5s error-tick +//! ratchet) compounds ×1.1 until gas×maxFee ≈ wallet balance. Cap +//! `max_fee_per_gas` at `CEIL = max(K × E.max_fee, ABS_FLOOR)` where `E` is +//! **this tick's** fresh estimate (Optimism txmgr shape: multiplier on the +//! suggestion, not an absolute-gwei knob). +//! +//! - `K = 3`: CEIL only clips once base has risen ~(2K−1)× since the estimate; +//! with a 72s re-estimate interval that is ~2.3× margin over max EIP-1559 +//! growth. Not an operator knob — a mis-set floor can cause a danger +//! shutdown with no validatable lower bound. +//! - `ABS_FLOOR` (2 gwei): on base≈0 devnet/L2, `3×E` can be a few wei and +//! sit forever under the node's min gas price. +//! - Clamp the **cap only** — the tip is steering; spend is still +//! `paid = min(cap, base+tip) ≤ CEIL`. +//! - Escape valve: if the clamp would put the cap below `bump(prior.cap)` +//! **and** `E.tip > prior.tip`, allow `cap = bump(prior.cap)` above CEIL +//! so a starved-tip prior can heal when tips recover (geth couples the +//! two fields). +//! - Hold: when the clamp bound this tick and the result cannot clear +//! +10% over `prior.cap`, still broadcast (mempool probe) but write no +//! floor — not an internal retry loop; the outer tick sleeps on the +//! confirmation cadence. +//! +//! Operator funding bound: `balance ≥ N_pending × gas × K × E.max_fee` +//! (≈ `N × gas × 6 × base` with the default estimator) plus `gas × CEIL` +//! free for `estimateGas`. use alloy::consensus::BlockHeader; use alloy::providers::{DynProvider, Provider, utils}; use alloy::rpc::types::BlockNumberOrTag; +/// Multiplier on this tick's estimated `max_fee_per_gas` for the replacement +/// ceiling. Documented constant next to the estimator pin — not a knob. +pub const FEE_CEILING_MULTIPLIER: u128 = 3; + +/// Absolute floor for the replacement ceiling (2 gwei). Keeps `3×E` above +/// typical node min-gas-price on base≈0 networks. +pub const FEE_CEILING_ABS_FLOOR_WEI: u128 = 2_000_000_000; + /// The three components of Alloy's default EIP-1559 estimate. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct Eip1559Fees { @@ -18,6 +56,31 @@ pub struct Eip1559Fees { pub max_fee_per_gas: u128, } +/// Result of [`fees_for_nonce`]: fees to send, and whether this tick is in +/// ceiling-hold (broadcast as probe, do not write the floor). +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub struct FeesForNonce { + pub fees: Eip1559Fees, + /// The ceiling clamped the cap and the result cannot clear + /// [`bumped_replacement_fees`] of `prior.max_fee`. Caller must still + /// broadcast, must not raise/replace the stored floor, and should + /// surface a Held tick outcome. + pub hold: bool, +} + +/// Pad an `eth_estimateGas` result so a tight estimate cannot mine as an +/// out-of-gas revert (burns the wallet-nonce slot with no `InputAdded`). +pub fn pad_gas_estimate(gas: u64) -> u64 { + gas.saturating_add(gas / 10) +} + +/// `CEIL = max(K × estimate_max_fee, ABS_FLOOR)`. +pub fn fee_ceiling(estimate_max_fee: u128) -> u128 { + FEE_CEILING_MULTIPLIER + .saturating_mul(estimate_max_fee) + .max(FEE_CEILING_ABS_FLOOR_WEI) +} + /// Bump one EIP-1559 component for a same-nonce replacement. /// /// ×1.1, plus 1 wei so integer division cannot stall on a flat spot and so @@ -39,10 +102,11 @@ fn bump_replacement_component(value: u128) -> u128 { /// the clamp is defense-in-depth for a one-shot bump of a near-zero-base /// estimate (the flusher) and for already-invalid inputs. /// -/// The poster floors a re-estimate against the last successful send at that -/// wallet nonce; the flusher bumps a fresh estimate so no-ops can compete -/// with pending batch txs. Eviction is operational acceleration, not a -/// correctness precondition. +/// The poster floors a re-estimate against the last recorded fees at that +/// wallet nonce (a successful broadcast **or** an underpriced raise that was +/// never accepted); the flusher prices no-ops at [`fee_ceiling`] so recovery +/// can clear anything the poster produced. Eviction is operational +/// acceleration, not a correctness precondition. pub fn bumped_replacement_fees(base_max_fee: u128, base_priority_fee: u128) -> (u128, u128) { let tip = base_priority_fee.min(base_max_fee); let new_max_fee = bump_replacement_component(base_max_fee); @@ -50,30 +114,67 @@ pub fn bumped_replacement_fees(base_max_fee: u128, base_priority_fee: u128) -> ( (new_max_fee, new_priority_fee) } -/// Absolute estimate, raised to a replacement floor when `prior` is set. +/// Absolute estimate, raised to a replacement floor when `prior` is set, then +/// capped by [`fee_ceiling`] on `max_fee_per_gas` only. +/// +/// First send at a nonce uses `estimate` (clamped so `tip ≤ max_fee`, then +/// ceiling). A same-nonce resubmit takes the per-component max of the fresh +/// estimate and [`bumped_replacement_fees`] of the last recorded fees, so a +/// flat market cannot re-broadcast underpriced replacements. The ceiling and +/// tip≤cap clamps run **after** that floor and **before** returning — never +/// clamp only at the call site (that recreates tip>cap). /// -/// First send at a nonce uses `estimate` (clamped so `tip ≤ max_fee`). A -/// same-nonce resubmit takes the per-component max of the fresh estimate and -/// [`bumped_replacement_fees`] of the last successful broadcast, so a flat -/// market cannot re-broadcast underpriced replacements. The tip is then -/// clamped to the fee cap: geth will not accept `maxPriorityFeePerGas > -/// maxFeePerGas`, and a rejected send must not become a sticky invalid pair. -pub fn fees_for_nonce(estimate: Eip1559Fees, prior: Option) -> Eip1559Fees { - let fees = match prior { - None => estimate, +/// Underpriced self-correction must call +/// `fees_for_nonce(estimate, Some(attempted))` — basing the raise on `E` +/// keeps the stored floor from parking at `1.1×CEIL`. +pub fn fees_for_nonce(estimate: Eip1559Fees, prior: Option) -> FeesForNonce { + let ceil = fee_ceiling(estimate.max_fee_per_gas); + + let (mut max_fee, mut tip) = match prior { + None => (estimate.max_fee_per_gas, estimate.max_priority_fee_per_gas), Some(prior) => { let (bumped_max, bumped_prio) = bumped_replacement_fees(prior.max_fee_per_gas, prior.max_priority_fee_per_gas); - Eip1559Fees { - base_fee_per_gas: estimate.base_fee_per_gas, - max_fee_per_gas: estimate.max_fee_per_gas.max(bumped_max), - max_priority_fee_per_gas: estimate.max_priority_fee_per_gas.max(bumped_prio), - } + ( + estimate.max_fee_per_gas.max(bumped_max), + estimate.max_priority_fee_per_gas.max(bumped_prio), + ) } }; - Eip1559Fees { - max_priority_fee_per_gas: fees.max_priority_fee_per_gas.min(fees.max_fee_per_gas), - ..fees + + let mut hold = false; + if max_fee > ceil { + let bump_prior_cap = prior.map(|p| bump_replacement_component(p.max_fee_per_gas)); + let escape = match (prior, bump_prior_cap) { + (Some(p), Some(bump_cap)) => { + // Clamp would put cap below the replacement floor, but tips + // recovered: allow one bump(prior.cap) step above CEIL so + // geth's coupled tip/cap rule can heal a starved-tip prior. + ceil < bump_cap && estimate.max_priority_fee_per_gas > p.max_priority_fee_per_gas + } + _ => false, + }; + if escape { + max_fee = bump_prior_cap.expect("escape implies prior"); + } else { + max_fee = ceil; + if let Some(p) = prior { + let (bump_cap, _) = + bumped_replacement_fees(p.max_fee_per_gas, p.max_priority_fee_per_gas); + // Clamp bound this tick and we still cannot clear +10%. + hold = max_fee < bump_cap; + } + } + } + + tip = tip.min(max_fee); + FeesForNonce { + fees: Eip1559Fees { + base_fee_per_gas: estimate.base_fee_per_gas, + max_fee_per_gas: max_fee, + max_priority_fee_per_gas: tip, + }, + hold, } } @@ -128,6 +229,23 @@ mod tests { assert_eq!(estimate.max_fee_per_gas, 204); } + #[test] + fn fee_ceiling_is_max_of_multiplier_and_abs_floor() { + assert_eq!(fee_ceiling(0), FEE_CEILING_ABS_FLOOR_WEI); + assert_eq!(fee_ceiling(1), FEE_CEILING_ABS_FLOOR_WEI); + assert_eq!( + fee_ceiling(10_000_000_000), + FEE_CEILING_MULTIPLIER * 10_000_000_000 + ); + } + + #[test] + fn pad_gas_estimate_adds_ten_percent() { + assert_eq!(pad_gas_estimate(100_000), 110_000); + assert_eq!(pad_gas_estimate(0), 0); + assert_eq!(pad_gas_estimate(u64::MAX), u64::MAX); + } + #[test] fn replacement_fee_bump_exceeds_ten_percent_for_max_fee() { for base in [1_u128, 10, 100, 1_000, 1_000_000, 1_000_000_000_000] { @@ -142,7 +260,6 @@ mod tests { #[test] fn replacement_fee_bump_exceeds_ten_percent_for_priority_fee() { for base in [1_u128, 10, 100, 1_000, 1_000_000, 1_000_000_000_000] { - // Cap high enough that the tip clamp does not bind. let (_, new_prio) = bumped_replacement_fees(base.saturating_mul(4), base); assert!( new_prio.saturating_mul(10) >= base.saturating_mul(11), @@ -184,14 +301,50 @@ mod tests { assert_eq!(new_prio, u128::MAX); } + fn assert_eip1559_valid(fees: Eip1559Fees) { + assert!( + fees.max_priority_fee_per_gas <= fees.max_fee_per_gas, + "invalid EIP-1559 pair: tip {} > max_fee {}", + fees.max_priority_fee_per_gas, + fees.max_fee_per_gas, + ); + } + #[test] - fn fees_for_nonce_passes_estimate_through_on_first_send() { + fn fees_for_nonce_passes_estimate_through_on_first_send_when_under_ceiling() { let estimate = Eip1559Fees { base_fee_per_gas: 100, max_priority_fee_per_gas: 2, max_fee_per_gas: 202, }; - assert_eq!(fees_for_nonce(estimate, None), estimate); + // 202 < ABS_FLOOR ≤ CEIL, so the upper clamp does not bind. + let out = fees_for_nonce(estimate, None); + assert!(!out.hold); + assert_eq!(out.fees, estimate); + } + + #[test] + fn fees_for_nonce_base_zero_ceiling_uses_abs_floor() { + // Tiny estimate → CEIL = ABS_FLOOR (not 3 wei). A prior above that + // clamps to the abs floor rather than a useless under-min-gas ceiling. + let estimate = Eip1559Fees { + base_fee_per_gas: 0, + max_priority_fee_per_gas: 1, + max_fee_per_gas: 1, + }; + assert_eq!( + fee_ceiling(estimate.max_fee_per_gas), + FEE_CEILING_ABS_FLOOR_WEI + ); + let prior = Eip1559Fees { + base_fee_per_gas: 0, + max_priority_fee_per_gas: 1, + max_fee_per_gas: FEE_CEILING_ABS_FLOOR_WEI, + }; + let out = fees_for_nonce(estimate, Some(prior)); + assert!(out.hold); + assert_eq!(out.fees.max_fee_per_gas, FEE_CEILING_ABS_FLOOR_WEI); + assert_eip1559_valid(out.fees); } #[test] @@ -201,15 +354,25 @@ mod tests { max_priority_fee_per_gas: 10, max_fee_per_gas: 1_000, }; - // Flat market: estimate equals prior. Replacement must clear ≥10%. - let estimate = prior; - let fees = fees_for_nonce(estimate, Some(prior)); + // Estimate high enough that 3×E clears bump(prior) and ABS_FLOOR. + let estimate = Eip1559Fees { + base_fee_per_gas: 100, + max_priority_fee_per_gas: 10, + max_fee_per_gas: 10_000_000_000, + }; + let out = fees_for_nonce(estimate, Some(prior)); let (bumped_max, bumped_prio) = bumped_replacement_fees(prior.max_fee_per_gas, prior.max_priority_fee_per_gas); - assert_eq!(fees.max_fee_per_gas, bumped_max); - assert_eq!(fees.max_priority_fee_per_gas, bumped_prio); - assert!(fees.max_fee_per_gas > prior.max_fee_per_gas); - assert!(fees.max_priority_fee_per_gas > prior.max_priority_fee_per_gas); + assert!(!out.hold); + assert_eq!( + out.fees.max_fee_per_gas, + estimate.max_fee_per_gas.max(bumped_max) + ); + assert_eq!( + out.fees.max_priority_fee_per_gas, + estimate.max_priority_fee_per_gas.max(bumped_prio) + ); + assert!(out.fees.max_fee_per_gas > prior.max_fee_per_gas); } #[test] @@ -222,118 +385,137 @@ mod tests { let estimate = Eip1559Fees { base_fee_per_gas: 500, max_priority_fee_per_gas: 50, - max_fee_per_gas: 10_000, + max_fee_per_gas: 10_000_000_000, }; - let fees = fees_for_nonce(estimate, Some(prior)); - assert_eq!(fees, estimate); + let out = fees_for_nonce(estimate, Some(prior)); + assert!(!out.hold); + assert_eq!(out.fees.max_fee_per_gas, estimate.max_fee_per_gas); + assert_eq!( + out.fees.max_priority_fee_per_gas, + estimate.max_priority_fee_per_gas + ); } #[test] - fn fees_for_nonce_clears_both_fields_when_estimate_is_mixed() { - // Market moved up on max_fee but not on priority (or the reverse): - // each component must still clear the ≥10% floor vs the in-flight tx. - let prior = Eip1559Fees { - base_fee_per_gas: 100, - max_priority_fee_per_gas: 100, - max_fee_per_gas: 1_000, + fn fees_for_nonce_iterated_retry_respects_ceiling_and_tip_cap() { + let estimate = Eip1559Fees { + base_fee_per_gas: 20_000_000_000, + max_priority_fee_per_gas: 1_000_000_000, + max_fee_per_gas: 41_000_000_000, }; - let (bumped_max, bumped_prio) = - bumped_replacement_fees(prior.max_fee_per_gas, prior.max_priority_fee_per_gas); + let ceil = fee_ceiling(estimate.max_fee_per_gas); + let mut fees = estimate; + for _ in 0..40 { + let out = fees_for_nonce(estimate, Some(fees)); + assert_eip1559_valid(out.fees); + assert!( + out.fees.max_fee_per_gas <= ceil + || out.fees.max_fee_per_gas == bump_replacement_component(fees.max_fee_per_gas), + "cap {} exceeded CEIL {ceil} without escape", + out.fees.max_fee_per_gas + ); + assert!(out.fees.max_priority_fee_per_gas <= out.fees.max_fee_per_gas); + if out.hold { + // Saturation: tip stays on the estimate/bump path, not ≈ CEIL. + assert!(out.fees.max_fee_per_gas <= ceil); + assert!( + out.fees.max_priority_fee_per_gas + <= estimate + .max_priority_fee_per_gas + .max(bump_replacement_component(fees.max_priority_fee_per_gas)) + .min(out.fees.max_fee_per_gas) + + 1, + "hold tip walked up toward CEIL: tip={}", + out.fees.max_priority_fee_per_gas + ); + break; + } + fees = out.fees; + assert!( + fees.max_fee_per_gas <= ceil, + "recorded floor must not exceed CEIL across raises" + ); + } + } - // High max_fee estimate, priority still below the replacement floor. - let estimate_high_max = Eip1559Fees { - base_fee_per_gas: 100, - max_priority_fee_per_gas: prior.max_priority_fee_per_gas + 1, // < 10% bump - max_fee_per_gas: bumped_max + 5_000, + #[test] + fn fees_for_nonce_hold_when_prior_above_ceiling_and_tips_flat() { + let estimate = Eip1559Fees { + base_fee_per_gas: 1_000_000_000, + max_priority_fee_per_gas: 1_000_000_000, + max_fee_per_gas: 10_000_000_000, // CEIL = 30 gwei }; - let fees = fees_for_nonce(estimate_high_max, Some(prior)); - assert_eq!(fees.max_fee_per_gas, estimate_high_max.max_fee_per_gas); - assert_eq!(fees.max_priority_fee_per_gas, bumped_prio); - - // High priority estimate, max_fee still below the replacement floor. - let estimate_high_prio = Eip1559Fees { - base_fee_per_gas: 100, - max_priority_fee_per_gas: bumped_prio + 50, - max_fee_per_gas: prior.max_fee_per_gas + 1, // < 10% bump + let ceil = fee_ceiling(estimate.max_fee_per_gas); + let prior = Eip1559Fees { + base_fee_per_gas: estimate.base_fee_per_gas, + max_priority_fee_per_gas: estimate.max_priority_fee_per_gas, + max_fee_per_gas: ceil, // already at ceiling }; - let fees = fees_for_nonce(estimate_high_prio, Some(prior)); - assert_eq!(fees.max_fee_per_gas, bumped_max); - assert_eq!( - fees.max_priority_fee_per_gas, - estimate_high_prio.max_priority_fee_per_gas - ); - } - - fn assert_eip1559_valid(fees: Eip1559Fees) { - assert!( - fees.max_priority_fee_per_gas <= fees.max_fee_per_gas, - "invalid EIP-1559 pair: tip {} > max_fee {}", - fees.max_priority_fee_per_gas, - fees.max_fee_per_gas, - ); - } - - fn assert_clears_replacement_floor(prior: Eip1559Fees, next: Eip1559Fees) { - assert!(next.max_fee_per_gas > prior.max_fee_per_gas); - assert!(next.max_priority_fee_per_gas > prior.max_priority_fee_per_gas); + let out = fees_for_nonce(estimate, Some(prior)); + assert!(out.hold, "flat tips at CEIL must hold"); + assert_eq!(out.fees.max_fee_per_gas, ceil); + assert_eip1559_valid(out.fees); + // Tip must not walk to CEIL on hold. + assert!(out.fees.max_priority_fee_per_gas < ceil); assert!( - next.max_fee_per_gas.saturating_mul(10) >= prior.max_fee_per_gas.saturating_mul(11), - "max_fee lost the ≥10% floor: prior={} next={}", - prior.max_fee_per_gas, - next.max_fee_per_gas, - ); - assert!( - next.max_priority_fee_per_gas.saturating_mul(10) - >= prior.max_priority_fee_per_gas.saturating_mul(11), - "priority lost the ≥10% floor: prior={} next={}", - prior.max_priority_fee_per_gas, - next.max_priority_fee_per_gas, + out.fees.max_priority_fee_per_gas + <= bump_replacement_component(prior.max_priority_fee_per_gas) ); } #[test] - fn fees_for_nonce_stays_valid_across_repeated_flat_retries() { - // The poster records the *sent* pair, so a stuck nonce compounds the - // bump against its own output. Asymmetric ×2 tip / ×1.1 cap crossed - // `tip > max_fee` in ~7 rounds at 20 gwei base / 1 gwei tip — and on - // the first retry when cap ≈ tip. Equal growth must stay valid. - for start in [ - Eip1559Fees { - base_fee_per_gas: 20_000_000_000, - max_priority_fee_per_gas: 1_000_000_000, - max_fee_per_gas: 41_000_000_000, - }, - Eip1559Fees { - base_fee_per_gas: 1, - max_priority_fee_per_gas: 1, - max_fee_per_gas: 1, - }, - Eip1559Fees { - base_fee_per_gas: 0, - max_priority_fee_per_gas: 1_000, - max_fee_per_gas: 1_000, - }, - ] { - let mut fees = start; - assert_eip1559_valid(fees); - for _ in 0..20 { - let next = fees_for_nonce(fees, Some(fees)); - assert_eip1559_valid(next); - assert_clears_replacement_floor(fees, next); - fees = next; - } - } + fn fees_for_nonce_escape_valve_heals_starved_tip_prior() { + let estimate = Eip1559Fees { + base_fee_per_gas: 10_000_000_000, + max_priority_fee_per_gas: 2_000_000_000, // recovered tip + max_fee_per_gas: 30_000_000_000, // CEIL = 90 gwei + }; + let ceil = fee_ceiling(estimate.max_fee_per_gas); + let prior = Eip1559Fees { + base_fee_per_gas: 10_000_000_000, + max_priority_fee_per_gas: 1, // starved (empty reward history) + max_fee_per_gas: ceil, + }; + let out = fees_for_nonce(estimate, Some(prior)); + let (bump_cap, _) = + bumped_replacement_fees(prior.max_fee_per_gas, prior.max_priority_fee_per_gas); + assert!(!out.hold, "recovered tip must escape hold"); + assert_eq!(out.fees.max_fee_per_gas, bump_cap); + assert!(out.fees.max_fee_per_gas > ceil); + assert!(out.fees.max_priority_fee_per_gas > prior.max_priority_fee_per_gas); + assert_eip1559_valid(out.fees); } #[test] fn fees_for_nonce_clamps_tip_above_fee_cap_on_first_send() { let estimate = Eip1559Fees { base_fee_per_gas: 1, - max_priority_fee_per_gas: 500, - max_fee_per_gas: 100, + max_priority_fee_per_gas: FEE_CEILING_ABS_FLOOR_WEI + 500, + max_fee_per_gas: FEE_CEILING_ABS_FLOOR_WEI, + }; + let out = fees_for_nonce(estimate, None); + assert_eq!(out.fees.max_fee_per_gas, FEE_CEILING_ABS_FLOOR_WEI); + assert_eq!(out.fees.max_priority_fee_per_gas, FEE_CEILING_ABS_FLOOR_WEI); + } + + #[test] + fn fees_for_nonce_underpriced_raise_baselines_on_estimate() { + // Raise must be fees_for_nonce(estimate, Some(attempted)), not + // fees_for_nonce(attempted, Some(attempted)) — otherwise the floor + // parks at 1.1×CEIL. + let estimate = Eip1559Fees { + base_fee_per_gas: 20_000_000_000, + max_priority_fee_per_gas: 1_000_000_000, + max_fee_per_gas: 41_000_000_000, + }; + let ceil = fee_ceiling(estimate.max_fee_per_gas); + let attempted = Eip1559Fees { + base_fee_per_gas: estimate.base_fee_per_gas, + max_priority_fee_per_gas: estimate.max_priority_fee_per_gas, + max_fee_per_gas: ceil, }; - let fees = fees_for_nonce(estimate, None); - assert_eq!(fees.max_fee_per_gas, 100); - assert_eq!(fees.max_priority_fee_per_gas, 100); + let raised = fees_for_nonce(estimate, Some(attempted)); + assert!(raised.hold || raised.fees.max_fee_per_gas <= ceil); + assert!(raised.fees.max_fee_per_gas <= ceil || raised.hold); } } diff --git a/sequencer/src/l1/submitter/config.rs b/sequencer/src/l1/submitter/config.rs index beddd9dd..1c0cbc65 100644 --- a/sequencer/src/l1/submitter/config.rs +++ b/sequencer/src/l1/submitter/config.rs @@ -15,10 +15,19 @@ use std::time::Duration; pub struct BatchSubmitterConfig { /// How often the submitter polls for new work when idle. pub idle_poll_interval_ms: u64, + /// Safe-confirmation depth used by the poster. + pub confirmation_depth: u64, + /// Assumed L1 block time used to pace fee-ceiling hold probes. + pub seconds_per_block: u64, } impl BatchSubmitterConfig { pub fn idle_poll_interval(&self) -> Duration { Duration::from_millis(self.idle_poll_interval_ms) } + + pub fn confirmation_cadence(&self) -> Duration { + let blocks = self.confirmation_depth.saturating_add(1).saturating_mul(2); + Duration::from_secs(blocks.saturating_mul(self.seconds_per_block)) + } } diff --git a/sequencer/src/l1/submitter/mod.rs b/sequencer/src/l1/submitter/mod.rs index 48178f8c..a313c7f7 100644 --- a/sequencer/src/l1/submitter/mod.rs +++ b/sequencer/src/l1/submitter/mod.rs @@ -15,5 +15,8 @@ mod poster; mod worker; pub use config::BatchSubmitterConfig; -pub use poster::{BatchPoster, BatchPosterConfig, BatchPosterError, EthereumBatchPoster, TxHash}; +pub use poster::{ + BatchPoster, BatchPosterConfig, BatchPosterError, EthereumBatchPoster, SubmitBatchesOutcome, + TxHash, +}; pub use worker::{BatchSubmitter, BatchSubmitterError, SubmitterExit}; diff --git a/sequencer/src/l1/submitter/poster.rs b/sequencer/src/l1/submitter/poster.rs index 783b7a40..48b41318 100644 --- a/sequencer/src/l1/submitter/poster.rs +++ b/sequencer/src/l1/submitter/poster.rs @@ -10,23 +10,19 @@ use async_trait::async_trait; use cartesi_rollups_contracts::input_box::InputBox; use sequencer_core::batch::Batch; use thiserror::Error; -use tracing::{debug, info, warn}; +use tracing::{debug, error, info, warn}; -use crate::l1::eip1559::{Eip1559Fees, estimate_fees, fees_for_nonce}; +use crate::l1::eip1559::{ + Eip1559Fees, FeesForNonce, estimate_fees, fees_for_nonce, pad_gas_estimate, +}; use crate::l1::partition::{decode_evm_advance_input, get_input_added_events_ordered}; use crate::l1::watermark::WalletNonceWatermarkSink; -use std::collections::BTreeMap; +use std::collections::{BTreeMap, BTreeSet}; use std::sync::{Arc, Mutex}; +use std::time::Instant; pub type TxHash = alloy_primitives::B256; -/// Pad an `eth_estimateGas` result so a tight estimate cannot mine as an -/// out-of-gas revert (which would burn the wallet-nonce slot with no -/// `InputAdded` and desynchronize payload↔nonce assignment). -fn pad_gas_estimate(gas: u64) -> u64 { - gas.saturating_add(gas / 10) -} - #[derive(Debug, Clone)] pub struct BatchPosterConfig { pub l1_submit_address: alloy_primitives::Address, @@ -55,6 +51,16 @@ pub enum BatchPosterError { ChainIdMismatch { rpc: u64, expected: u64 }, } +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum SubmitBatchesOutcome { + /// All sends progressed normally. + Submitted(Vec), + /// At least one nonce is in fee-ceiling hold: still broadcast as a mempool + /// probe and wrote no floor. This is not an internal retry loop; the outer + /// tick sleeps on the confirmation cadence. + Held(Vec), +} + #[async_trait] pub trait BatchPoster: Send + Sync { /// Broadcast the payloads as L1 txs at consecutive wallet nonces. @@ -65,7 +71,7 @@ pub trait BatchPoster: Send + Sync { &self, payloads: Vec>, watermark: &dyn WalletNonceWatermarkSink, - ) -> Result, BatchPosterError>; + ) -> Result; async fn observed_submitted_batch_nonces( &self, @@ -93,6 +99,13 @@ pub struct EthereumBatchPoster { /// underpriced" still raises the stored floor so the next tick self-corrects /// without waiting for a confirmation timeout. in_flight: Arc>>, + /// Nonces already reported as entering ceiling hold. Cleared when Latest + /// advances or the nonce leaves hold, so operators see transitions rather + /// than one warning per tick. + ceiling_holds: Arc>>, + /// Last insufficient-funds operator alert. Provider failures still return + /// every time; only the duplicate log is rate-limited. + last_insufficient_funds_log: Arc>>, /// Test-only: next `send_batch_at_nonce` returns this error string without /// broadcasting (so callers can assert map updates / underpriced handling). #[cfg(test)] @@ -105,6 +118,8 @@ impl EthereumBatchPoster { provider, config, in_flight: Arc::new(Mutex::new(BTreeMap::new())), + ceiling_holds: Arc::new(Mutex::new(BTreeSet::new())), + last_insufficient_funds_log: Arc::new(Mutex::new(None)), #[cfg(test)] fail_next_send: Arc::new(Mutex::new(None)), } @@ -141,6 +156,45 @@ impl EthereumBatchPoster { ) } + fn note_ceiling_hold(&self, nonce: u64, fees: Eip1559Fees) { + let first = self + .ceiling_holds + .lock() + .expect("ceiling_holds lock") + .insert(nonce); + if first { + warn!( + tx_nonce = nonce, + max_fee_per_gas = fees.max_fee_per_gas, + max_priority_fee_per_gas = fees.max_priority_fee_per_gas, + "batch submission entered fee-ceiling hold; probing again on confirmation cadence" + ); + } + } + + fn clear_ceiling_hold(&self, nonce: u64) { + self.ceiling_holds + .lock() + .expect("ceiling_holds lock") + .remove(&nonce); + } + + fn log_insufficient_funds(&self) { + let now = Instant::now(); + let mut last = self + .last_insufficient_funds_log + .lock() + .expect("last_insufficient_funds_log lock"); + if last.is_some_and(|then| now.duration_since(then) < self.confirmation_timeout()) { + return; + } + *last = Some(now); + error!( + submitter_address = %self.config.batch_submitter_address, + "batch submitter has insufficient funds; top up the batch-submitter wallet" + ); + } + async fn latest_account_nonce(&self) -> Result { self.provider .get_transaction_count(self.config.batch_submitter_address) @@ -198,22 +252,10 @@ impl EthereumBatchPoster { /// Wait serially for each tx to reach `confirmation_depth + 1` confirmations. /// - /// **Serial is not a performance concession; it's correct.** Ethereum mines - /// transactions from a single EOA in strict wallet-nonce order: tx[k] cannot - /// land on-chain until tx[k-1] has landed. So: - /// - /// - If tx[0] times out, tx[1..] cannot have been mined either; watching - /// them is provably pointless. We return `Ok(())` early and let the next - /// tick retry the whole sequence. - /// - If tx[0] confirms, tx[1] was blocked only on tx[0] and is unblocked by - /// the time we start watching it. - /// /// Timeouts return `Ok(())` rather than `Err` because the safe response is - /// "re-enter `submit_batches` on the next tick" — which re-estimates fees, - /// floors **every** still-pending nonce against its in-flight record, and - /// at-least-once re-broadcasts the whole unconfirmed suffix. The - /// wallet-nonce ordering invariant above guarantees we cannot accidentally - /// skip work by returning early here. + /// "re-enter `submit_batches` on the next tick." That tick re-derives the + /// unresolved suffix from Latest, so returning after the first timeout is + /// safe even if a replacement caused a later watched hash to mine. async fn wait_for_confirmations(&self, tx_hashes: &[TxHash]) -> Result<(), BatchPosterError> { let timeout = self.confirmation_timeout(); for tx_hash in tx_hashes { @@ -258,6 +300,18 @@ fn is_replacement_underpriced(err: &str) -> bool { .contains("replacement transaction underpriced") } +fn is_already_known(err: &str) -> bool { + let err = err.to_ascii_lowercase(); + err.contains("already known") + || err.contains("already imported") + || err.contains("known transaction") +} + +fn is_insufficient_funds(err: &str) -> bool { + let err = err.to_ascii_lowercase(); + err.contains("insufficient funds") || err.contains("gas required exceeds allowance") +} + fn derive_confirmation_timeout( confirmation_depth: u64, seconds_per_block: u64, @@ -272,9 +326,9 @@ impl BatchPoster for EthereumBatchPoster { &self, payloads: Vec>, watermark: &dyn WalletNonceWatermarkSink, - ) -> Result, BatchPosterError> { + ) -> Result { if payloads.is_empty() { - return Ok(Vec::new()); + return Ok(SubmitBatchesOutcome::Submitted(Vec::new())); } // Keyed-write chain-id gate (review): re-confirm the RPC still serves the @@ -309,6 +363,10 @@ impl BatchPoster for EthereumBatchPoster { let mut in_flight = self.in_flight.lock().expect("in_flight lock"); in_flight.retain(|&nonce, _| nonce >= next_nonce); } + self.ceiling_holds + .lock() + .expect("ceiling_holds lock") + .retain(|&nonce| nonce >= next_nonce); // Write-before-broadcast (R1a): durably cover every nonce this // tick will use before the first send. One raise to the highest @@ -319,24 +377,56 @@ impl BatchPoster for EthereumBatchPoster { .map_err(BatchPosterError::Provider)?; let mut tx_hashes = Vec::with_capacity(payloads.len()); + let mut any_held = false; for payload in payloads { let prior = { let in_flight = self.in_flight.lock().expect("in_flight lock"); in_flight.get(&next_nonce).copied() }; - let fees = fees_for_nonce(estimate, prior); + let FeesForNonce { fees, hold } = fees_for_nonce(estimate, prior); let pending = match self.send_batch_at_nonce(payload, next_nonce, &fees).await { - Ok(pending) => pending, + Ok(pending) => { + if hold { + any_held = true; + self.note_ceiling_hold(next_nonce, fees); + } else { + self.clear_ceiling_hold(next_nonce); + self.in_flight + .lock() + .expect("in_flight lock") + .insert(next_nonce, fees); + } + pending + } + Err(BatchPosterError::Provider(ref msg)) if is_already_known(msg) => { + self.clear_ceiling_hold(next_nonce); + self.in_flight + .lock() + .expect("in_flight lock") + .insert(next_nonce, fees); + next_nonce = next_nonce.saturating_add(1); + continue; + } Err(BatchPosterError::Provider(ref msg)) if is_replacement_underpriced(msg) => { + if hold { + any_held = true; + self.note_ceiling_hold(next_nonce, fees); + next_nonce = next_nonce.saturating_add(1); + continue; + } // Node rejected the replacement fee — raise the floor from // what we just tried so the next tick clears the threshold // without waiting for a confirmation timeout. - let raised = fees_for_nonce(fees, Some(fees)); + let raised = fees_for_nonce(estimate, Some(fees)); self.in_flight .lock() .expect("in_flight lock") - .insert(next_nonce, raised); + .insert(next_nonce, raised.fees); + return Err(BatchPosterError::Provider(msg.clone())); + } + Err(BatchPosterError::Provider(ref msg)) if is_insufficient_funds(msg) => { + self.log_insufficient_funds(); return Err(BatchPosterError::Provider(msg.clone())); } Err(err) => return Err(err), @@ -346,10 +436,6 @@ impl BatchPoster for EthereumBatchPoster { // underpriced path above, which self-corrects against a live pending // tx the node already holds). let tx_hash = *pending.tx_hash(); - self.in_flight - .lock() - .expect("in_flight lock") - .insert(next_nonce, fees); debug!( tx_nonce = next_nonce, %tx_hash, @@ -363,7 +449,11 @@ impl BatchPoster for EthereumBatchPoster { } self.wait_for_confirmations(tx_hashes.as_slice()).await?; - Ok(tx_hashes) + if any_held { + Ok(SubmitBatchesOutcome::Held(tx_hashes)) + } else { + Ok(SubmitBatchesOutcome::Submitted(tx_hashes)) + } } async fn observed_submitted_batch_nonces( @@ -419,7 +509,7 @@ impl BatchPoster for EthereumBatchPoster { #[cfg(test)] pub(crate) mod mock { - use super::{Batch, BatchPoster, BatchPosterError, TxHash}; + use super::{Batch, BatchPoster, BatchPosterError, SubmitBatchesOutcome, TxHash}; use crate::l1::watermark::WalletNonceWatermarkSink; use async_trait::async_trait; use std::sync::Mutex; @@ -430,6 +520,7 @@ pub(crate) mod mock { pub observed_submitted_nonces: Mutex>, pub observed_submitted_error: Mutex>, pub last_from_block: Mutex>, + pub held: Mutex, } impl MockBatchPoster { @@ -439,6 +530,7 @@ pub(crate) mod mock { observed_submitted_nonces: Mutex::new(Vec::new()), observed_submitted_error: Mutex::new(None), last_from_block: Mutex::new(None), + held: Mutex::new(false), } } @@ -457,6 +549,10 @@ pub(crate) mod mock { pub fn last_from_block(&self) -> Option { *self.last_from_block.lock().expect("lock") } + + pub fn set_held(&self, held: bool) { + *self.held.lock().expect("lock") = held; + } } #[async_trait] @@ -465,7 +561,7 @@ pub(crate) mod mock { &self, payloads: Vec>, _watermark: &dyn WalletNonceWatermarkSink, - ) -> Result, BatchPosterError> { + ) -> Result { let mut tx_hashes = Vec::with_capacity(payloads.len()); for payload in payloads { let batch_index = ssz::Decode::from_ssz_bytes(payload.as_ref()) @@ -477,7 +573,11 @@ pub(crate) mod mock { .push((batch_index, payload.len())); tx_hashes.push(TxHash::ZERO); } - Ok(tx_hashes) + if *self.held.lock().expect("lock") { + Ok(SubmitBatchesOutcome::Held(tx_hashes)) + } else { + Ok(SubmitBatchesOutcome::Submitted(tx_hashes)) + } } async fn observed_submitted_batch_nonces( @@ -511,14 +611,24 @@ mod tests { use super::{ BatchPoster, BatchPosterConfig, BatchPosterError, EthereumBatchPoster, - derive_confirmation_timeout, is_replacement_underpriced, mock::MockBatchPoster, - pad_gas_estimate, + SubmitBatchesOutcome, derive_confirmation_timeout, is_already_known, is_insufficient_funds, + is_replacement_underpriced, mock::MockBatchPoster, }; + use crate::l1::eip1559::{fee_ceiling, pad_gas_estimate}; use crate::l1::watermark::WalletNonceWatermarkSink; use alloy::node_bindings::Anvil; use alloy::providers::Provider; use alloy::rpc::types::BlockNumberOrTag; + fn submitted_hashes(outcome: SubmitBatchesOutcome) -> Vec { + match outcome { + SubmitBatchesOutcome::Submitted(hashes) => hashes, + SubmitBatchesOutcome::Held(hashes) => { + panic!("expected Submitted outcome, got Held({hashes:?})") + } + } + } + fn require_anvil() { assert!( std::process::Command::new("anvil") @@ -734,17 +844,37 @@ mod tests { )); } + #[test] + fn already_known_matches_common_client_wording_case_insensitively() { + assert!(is_already_known("already known")); + assert!(is_already_known("Already Imported")); + assert!(is_already_known("Known transaction")); + assert!(!is_already_known("nonce too low")); + } + + #[test] + fn insufficient_funds_matches_common_client_wording_case_insensitively() { + assert!(is_insufficient_funds( + "insufficient funds for gas * price + value" + )); + assert!(is_insufficient_funds("Gas required exceeds allowance")); + assert!(!is_insufficient_funds( + "replacement transaction underpriced" + )); + } + #[test] fn underpriced_send_raises_floor_from_attempted_fees() { - let attempted = crate::l1::eip1559::Eip1559Fees { + let estimate = crate::l1::eip1559::Eip1559Fees { base_fee_per_gas: 20_000_000_000, max_priority_fee_per_gas: 1_000_000_000, max_fee_per_gas: 41_000_000_000, }; - let raised = crate::l1::eip1559::fees_for_nonce(attempted, Some(attempted)); - assert!(raised.max_fee_per_gas > attempted.max_fee_per_gas); - assert!(raised.max_priority_fee_per_gas > attempted.max_priority_fee_per_gas); - assert!(raised.max_priority_fee_per_gas <= raised.max_fee_per_gas); + let attempted = crate::l1::eip1559::fees_for_nonce(estimate, None).fees; + let raised = crate::l1::eip1559::fees_for_nonce(estimate, Some(attempted)); + assert!(raised.fees.max_fee_per_gas > attempted.max_fee_per_gas); + assert!(raised.fees.max_priority_fee_per_gas > attempted.max_priority_fee_per_gas); + assert!(raised.fees.max_priority_fee_per_gas <= raised.fees.max_fee_per_gas); } fn poster_config(anvil: &alloy::node_bindings::AnvilInstance) -> BatchPosterConfig { @@ -783,11 +913,18 @@ mod tests { .await .expect("base nonce"); // Prior fees high enough that a fresh Anvil estimate will not clear the - // ≥10% floor on its own — the poster must bump against this record. + // ≥10% floor on its own, but below this estimate's ceiling so this is a + // normal replacement rather than a ceiling hold. + let estimate = crate::l1::eip1559::estimate_fees(&provider) + .await + .expect("fee estimate"); let prior = crate::l1::eip1559::Eip1559Fees { - base_fee_per_gas: 1, - max_priority_fee_per_gas: 50_000_000, // 0.05 gwei - max_fee_per_gas: 100_000_000_000, // 100 gwei + base_fee_per_gas: estimate.base_fee_per_gas, + max_priority_fee_per_gas: estimate + .max_priority_fee_per_gas + .saturating_mul(2) + .min(estimate.max_fee_per_gas.saturating_mul(2)), + max_fee_per_gas: estimate.max_fee_per_gas.saturating_mul(2), }; poster.seed_in_flight_fees_for_test(BTreeMap::from([(base_nonce, prior)])); @@ -843,10 +980,12 @@ mod tests { .expect("base nonce"); // 1) Original poster tx parks in the mempool (mining disabled). - let first_hashes = poster - .submit_batches(vec![vec![0u8; 4]], &sink) - .await - .expect("first submit parks a pending tx"); + let first_hashes = submitted_hashes( + poster + .submit_batches(vec![vec![0u8; 4]], &sink) + .await + .expect("first submit parks a pending tx"), + ); assert_eq!(first_hashes.len(), 1); let first_hash = first_hashes[0]; @@ -878,10 +1017,12 @@ mod tests { // 2) Confirmation watch timed out; next tick must bump fees and // broadcast a same-nonce replacement (the gas-estimate fix). - let second_hashes = poster - .submit_batches(vec![vec![0u8; 4]], &sink) - .await - .expect("replacement must clear gas estimation and broadcast"); + let second_hashes = submitted_hashes( + poster + .submit_batches(vec![vec![0u8; 4]], &sink) + .await + .expect("replacement must clear gas estimation and broadcast"), + ); assert_eq!(second_hashes.len(), 1); let replacement_hash = second_hashes[0]; assert_ne!( @@ -1089,7 +1230,7 @@ mod tests { let prior = crate::l1::eip1559::Eip1559Fees { base_fee_per_gas: 1, max_priority_fee_per_gas: 50_000_000, - max_fee_per_gas: 100_000_000_000, + max_fee_per_gas: 1_000_000_000, }; poster.seed_in_flight_fees_for_test(BTreeMap::from([(base_nonce, prior)])); poster.fail_next_send_with_for_test( @@ -1121,6 +1262,44 @@ mod tests { assert!(raised.max_priority_fee_per_gas <= raised.max_fee_per_gas); } + #[tokio::test] + async fn submit_batches_underpriced_at_ceiling_holds_without_raising_floor() { + require_anvil(); + let anvil = Anvil::default().arg("--no-mining").timeout(30_000).spawn(); + let key = "0xac0974bec39a17e36ba4a6b4d238ff944bacb478cbed5efcae784d7bf4f2ff80"; + let submitter = alloy_primitives::address!("0xf39Fd6e51aad88F6F4ce6aB8827279cffFb92266"); + let provider = crate::l1::provider::create_signer_provider(&anvil.endpoint(), key, false) + .expect("signer provider"); + let poster = EthereumBatchPoster::new(provider.clone(), poster_config(&anvil)); + let sink = RecordingWatermarkSink::passing(); + + let base_nonce = provider + .get_transaction_count(submitter) + .await + .expect("base nonce"); + let estimate = crate::l1::eip1559::estimate_fees(&provider) + .await + .expect("fee estimate"); + let seed = crate::l1::eip1559::Eip1559Fees { + max_fee_per_gas: fee_ceiling(estimate.max_fee_per_gas), + ..estimate + }; + let seeded = BTreeMap::from([(base_nonce, seed)]); + poster.seed_in_flight_fees_for_test(seeded.clone()); + poster.fail_next_send_with_for_test("Replacement transaction underpriced"); + + let outcome = poster + .submit_batches(vec![vec![0u8; 4]], &sink) + .await + .expect("ceiling hold is a non-error tick outcome"); + assert_eq!(outcome, SubmitBatchesOutcome::Held(Vec::new())); + assert_eq!( + poster.in_flight_fees_for_test(), + seeded, + "ceiling hold must leave the stored floor byte-identical" + ); + } + /// Restart shape: empty in-flight map vs a live pending tx. Unconditional /// nonce-free gas estimate must still reach `eth_sendRawTransaction`. #[tokio::test] @@ -1133,40 +1312,29 @@ mod tests { let poster = EthereumBatchPoster::new(provider.clone(), poster_config(&anvil)); let sink = RecordingWatermarkSink::passing(); - let first_hashes = poster - .submit_batches(vec![vec![0u8; 4]], &sink) - .await - .expect("first submit parks a pending tx"); - let first_hash = first_hashes[0]; + let first_hashes = submitted_hashes( + poster + .submit_batches(vec![vec![0u8; 4]], &sink) + .await + .expect("first submit parks a pending tx"), + ); + assert_eq!(first_hashes.len(), 1); // Simulate restart / lost response: forget the floor while the tx is // still pending on the node. poster.seed_in_flight_fees_for_test(BTreeMap::new()); - let result = poster.submit_batches(vec![vec![0u8; 4]], &sink).await; - match result { - Ok(second_hashes) => { - assert_ne!( - second_hashes[0], first_hash, - "if fees moved, the untracked replacement must be a new hash" - ); - } - Err(BatchPosterError::Provider(msg)) => { - let lower = msg.to_ascii_lowercase(); - assert!( - !lower.contains("nonce too low"), - "untracked replacement must not die at estimateGas: {msg}" - ); - // Without a floor the re-estimate can match the pending tx - // exactly ("already imported") or land underpriced — either - // proves we reached send, which is the restart-shape hole. - assert!( - lower.contains("already imported") || is_replacement_underpriced(&msg), - "unexpected send failure for untracked replacement: {msg}" - ); - } - Err(other) => panic!("unexpected error: {other:?}"), - } + let outcome = poster + .submit_batches(vec![vec![0u8; 4]], &sink) + .await + .expect("already-known/imported is successful at-least-once progress"); + let SubmitBatchesOutcome::Submitted(second_hashes) = outcome else { + panic!("untracked first-fee retry cannot enter ceiling hold"); + }; + assert!( + second_hashes.len() <= 1, + "already-known sends may omit the hash; fresh replacements return one" + ); } /// Multi-nonce suffix: every pending nonce is re-broadcast and floored on @@ -1188,18 +1356,22 @@ mod tests { .expect("base nonce"); let payloads = vec![vec![0u8; 4], vec![1u8; 4], vec![2u8; 4]]; - let first_hashes = poster - .submit_batches(payloads.clone(), &sink) - .await - .expect("first multi-nonce submit"); + let first_hashes = submitted_hashes( + poster + .submit_batches(payloads.clone(), &sink) + .await + .expect("first multi-nonce submit"), + ); assert_eq!(first_hashes.len(), 3); let prior_fees = poster.in_flight_fees_for_test(); assert_eq!(prior_fees.len(), 3); - let second_hashes = poster - .submit_batches(payloads, &sink) - .await - .expect("suffix rebroadcast"); + let second_hashes = submitted_hashes( + poster + .submit_batches(payloads, &sink) + .await + .expect("suffix rebroadcast"), + ); assert_eq!(second_hashes.len(), 3); for (first, second) in first_hashes.iter().zip(second_hashes.iter()) { assert_ne!( diff --git a/sequencer/src/l1/submitter/worker.rs b/sequencer/src/l1/submitter/worker.rs index 5ea17639..60c07554 100644 --- a/sequencer/src/l1/submitter/worker.rs +++ b/sequencer/src/l1/submitter/worker.rs @@ -17,8 +17,8 @@ //! //! The outer loop is uniform: tick, maybe sleep, repeat. A tick that produced //! submissions re-enters immediately (no sleep) so the suffix drains quickly; -//! an idle or transient-error tick sleeps `idle_poll_interval` before the next -//! attempt. +//! an idle or transient-error tick sleeps `idle_poll_interval`, while a +//! fee-ceiling hold sleeps the confirmation cadence before the next probe. //! //! Mid-tick cancellation is crash-safe: storage transactions either commit or //! auto-roll-back on drop, and any already-sent L1 transaction is picked up by @@ -30,7 +30,9 @@ use std::time::Duration; use thiserror::Error; use tracing::{debug, error}; -use crate::l1::submitter::{BatchPoster, BatchPosterError, BatchSubmitterConfig}; +use crate::l1::submitter::{ + BatchPoster, BatchPosterError, BatchSubmitterConfig, SubmitBatchesOutcome, +}; use crate::runtime::shutdown::ShutdownSignal; use crate::storage::{PendingBatch, Storage, StorageOpenError, SubmitterFrontier}; @@ -65,6 +67,9 @@ pub(crate) enum TickOutcome { /// Submitted one or more batches; re-enter immediately so the suffix /// drains without idle-sleep. Submitted(usize), + /// Fee ceiling prevented a valid replacement floor. This is not an + /// internal retry loop; the outer loop waits on confirmation cadence. + Held, /// Transient provider error; log and sleep before retrying. Transient, } @@ -88,6 +93,7 @@ pub struct BatchSubmitter { db_path: String, poster: Arc

, idle_poll_interval: Duration, + confirmation_cadence: Duration, /// Write-before-broadcast hook (review R1a): the poster raises the /// persisted wallet-nonce watermark through this before every send. watermark_sink: crate::l1::watermark::StorageWatermarkSink, @@ -101,6 +107,7 @@ impl BatchSubmitter

{ db_path, poster, idle_poll_interval: config.idle_poll_interval(), + confirmation_cadence: config.confirmation_cadence(), } } @@ -136,8 +143,8 @@ impl BatchSubmitter

{ } /// Tick → sleep-if-idle → tick. Productive ticks re-enter immediately; - /// idle or transient-error ticks wait `idle_poll_interval`. Fatal errors - /// propagate. + /// idle or transient-error ticks wait `idle_poll_interval`; held ticks + /// wait the confirmation cadence. Fatal errors propagate. async fn run_loop(&self) -> Result { loop { let outcome = match self.tick_once().await { @@ -156,6 +163,9 @@ impl BatchSubmitter

{ }; match outcome { TickOutcome::Submitted(_) => continue, + TickOutcome::Held => { + tokio::time::sleep(self.confirmation_cadence).await; + } TickOutcome::Idle | TickOutcome::Transient => { tokio::time::sleep(self.idle_poll_interval).await; } @@ -191,20 +201,34 @@ impl BatchSubmitter

{ } let submitted_count = pending.len(); let payloads: Vec> = pending.into_iter().map(|b| b.encoded).collect(); - let tx_hashes = self + let outcome = self .poster .submit_batches(payloads, &self.watermark_sink) .await?; - if tx_hashes.len() != submitted_count { - return Err(BatchSubmitterError::Poster(BatchPosterError::Provider( - format!( - "poster returned {} tx hashes for {submitted_count} submitted batches", - tx_hashes.len(), - ), - ))); + match outcome { + SubmitBatchesOutcome::Submitted(tx_hashes) => { + if tx_hashes.len() != submitted_count { + return Err(BatchSubmitterError::Poster(BatchPosterError::Provider( + format!( + "poster returned {} tx hashes for {submitted_count} submitted batches", + tx_hashes.len(), + ), + ))); + } + Ok(TickOutcome::Submitted(submitted_count)) + } + SubmitBatchesOutcome::Held(tx_hashes) => { + if tx_hashes.len() > submitted_count { + return Err(BatchSubmitterError::Poster(BatchPosterError::Provider( + format!( + "held poster returned {} tx hashes for {submitted_count} submitted batches", + tx_hashes.len(), + ), + ))); + } + Ok(TickOutcome::Held) + } } - - Ok(TickOutcome::Submitted(submitted_count)) } async fn load_frontier(&self) -> Result { @@ -263,6 +287,8 @@ mod tests { fn default_test_config() -> BatchSubmitterConfig { BatchSubmitterConfig { idle_poll_interval_ms: 1000, + confirmation_depth: 0, + seconds_per_block: 1, } } @@ -327,6 +353,21 @@ mod tests { assert_eq!(submissions[2].0, 2); } + #[tokio::test] + async fn tick_once_surfaces_fee_ceiling_hold() { + let TestDb { _dir, path } = temp_db("tick-held"); + seed_two_closed_batches(&path); + + let mock = Arc::new(MockBatchPoster::new()); + mock.set_held(true); + let submitter = + super::BatchSubmitter::new(path.clone(), mock.clone(), default_test_config()); + + let outcome = submitter.tick_once().await.expect("tick once"); + assert_eq!(outcome, TickOutcome::Held); + assert_eq!(mock.submissions().len(), 3); + } + #[tokio::test] async fn tick_once_submits_nothing_when_already_caught_up() { let TestDb { _dir, path } = temp_db("tick-caught-up"); diff --git a/sequencer/src/recovery/flusher.rs b/sequencer/src/recovery/flusher.rs index f70944ff..576ecc85 100644 --- a/sequencer/src/recovery/flusher.rs +++ b/sequencer/src/recovery/flusher.rs @@ -19,7 +19,7 @@ use std::time::Duration; use thiserror::Error; use tracing::{debug, error, info}; -use crate::l1::eip1559::bumped_replacement_fees; +use crate::l1::eip1559::{bumped_replacement_fees, estimate_fees, fee_ceiling, pad_gas_estimate}; use crate::l1::watermark::{StorageWatermarkSink, WalletNonceWatermarkSink}; #[derive(Debug, Error)] @@ -239,57 +239,48 @@ impl MempoolFlusher { return Ok(Vec::new()); } - let fees = self + let estimate = estimate_fees(&self.provider) + .await + .map_err(FlushError::Provider)?; + let max_fee = fee_ceiling(estimate.max_fee_per_gas); + let (_, bumped_priority_fee) = + bumped_replacement_fees(estimate.max_fee_per_gas, estimate.max_priority_fee_per_gas); + let priority_fee = bumped_priority_fee.min(max_fee); + + // Estimate once without a nonce, explicitly at Latest. This mirrors + // the poster's nonce-free estimate and avoids Pending nonce policy; + // gas is loop-invariant for identical self-transfer no-ops. + let tx_base = alloy::rpc::types::TransactionRequest::default() + .with_from(self.address) + .with_to(self.address) + .with_value(U256::ZERO); + let gas = self .provider - .estimate_eip1559_fees() + .estimate_gas(tx_base.clone()) + .block(BlockNumberOrTag::Latest.into()) .await + .map(pad_gas_estimate) .map_err(|e| FlushError::Provider(e.to_string()))?; - // Bump the absolute estimate so no-ops can compete with pending batch - // txs at the same wallet nonces (shared ≥10% replacement helper — - // symmetric ×1.1+1 on both EIP-1559 components). Safety does not - // depend on the no-op winning — `flush_and_wait` only returns once - // Pending ≤ Safe. - // - // Residual gap: this is a one-shot bump of a *fresh* estimate, not of - // the pending tx's fees, so a replacement can still be underpriced - // when the two EIP-1559 components have moved independently. A - // rejected no-op hard-errors `flush_and_wait`; the orchestrator - // respawn retries. Tightening that needs the pending tx's fees (or a - // raise-on-underpriced-error loop), not a bigger one-shot multiplier. - let (bumped_max_fee, bumped_priority_fee) = - bumped_replacement_fees(fees.max_fee_per_gas, fees.max_priority_fee_per_gas); - debug!( from_nonce, to_nonce, count = to_nonce - from_nonce, - max_fee_per_gas = bumped_max_fee, - max_priority_fee = bumped_priority_fee, + max_fee_per_gas = max_fee, + max_priority_fee = priority_fee, + gas, "submitting flush no-ops" ); let mut tx_hashes = Vec::new(); let mut send_failures = Vec::new(); for nonce in from_nonce..to_nonce { - // Estimate without the pending nonce and pin gas (+10% pad) — - // same Anvil "nonce too low" hole as the batch poster. With gas - // + both fee fields set, the GasFiller skips a second estimate. - let tx_base = alloy::rpc::types::TransactionRequest::default() - .with_to(self.address) - .with_value(U256::ZERO) - .with_max_fee_per_gas(bumped_max_fee) - .with_max_priority_fee_per_gas(bumped_priority_fee); - let gas = match self.provider.estimate_gas(tx_base.clone()).await { - Ok(gas) => gas.saturating_add(gas / 10), - Err(e) => { - let message = e.to_string(); - error!(nonce, error = %message, "flush no-op gas estimate failed"); - send_failures.push((nonce, message)); - continue; - } - }; - let tx = tx_base.with_gas_limit(gas).with_nonce(nonce); + let tx = tx_base + .clone() + .with_gas_limit(gas) + .with_nonce(nonce) + .with_max_fee_per_gas(max_fee) + .with_max_priority_fee_per_gas(priority_fee); match self.provider.send_transaction(tx).await { Ok(pending) => { @@ -379,11 +370,9 @@ mod tests { } } - // ── H5: replacement-fee bump keeps no-ops competitive ───────── - // Rule itself lives in `l1::eip1559` (shared with the poster); the - // flusher's use site is the `bumped_replacement_fees(...)` call in - // `submit_noops`. Equal ×1.1 growth plus the tip≤cap clamp are what - // keep a one-shot bump of a near-zero-base estimate valid. + // ── H5: ceiling pricing keeps no-ops competitive ────────────── + // The flusher uses the poster's shared ceiling for max fee and a bumped, + // cap-clamped priority fee, so it can clear any ordinary poster floor. #[test] fn send_failure_error_summarizes_failed_slots() { diff --git a/sequencer/src/runtime/workers.rs b/sequencer/src/runtime/workers.rs index 4e948e08..54d163f9 100644 --- a/sequencer/src/runtime/workers.rs +++ b/sequencer/src/runtime/workers.rs @@ -178,6 +178,8 @@ impl Workers { let poster = Arc::new(EthereumBatchPoster::new(provider, poster_config)); let submitter_config = BatchSubmitterConfig { idle_poll_interval_ms: run_config.batch_submitter_idle_poll_interval_ms, + confirmation_depth: run_config.batch_submitter_confirmation_depth, + seconds_per_block: run_config.timing.seconds_per_block, }; let submitter = BatchSubmitter::new(db_path.clone(), poster, submitter_config) .start(shutdown.clone())?; diff --git a/sequencer/tests/batch_submitter_integration.rs b/sequencer/tests/batch_submitter_integration.rs index c2906b6d..635e957b 100644 --- a/sequencer/tests/batch_submitter_integration.rs +++ b/sequencer/tests/batch_submitter_integration.rs @@ -8,7 +8,7 @@ use std::sync::Mutex; use std::time::Duration; use async_trait::async_trait; -use sequencer::l1::submitter::{BatchPoster, BatchPosterError, TxHash}; +use sequencer::l1::submitter::{BatchPoster, BatchPosterError, SubmitBatchesOutcome, TxHash}; use sequencer::l1::submitter::{BatchSubmitter, BatchSubmitterConfig}; use sequencer::l1::watermark::WalletNonceWatermarkSink; use sequencer::runtime::shutdown::ShutdownSignal; @@ -62,7 +62,7 @@ impl BatchPoster for TestMock { &self, payloads: Vec>, _watermark: &dyn WalletNonceWatermarkSink, - ) -> Result, BatchPosterError> { + ) -> Result { // Transient-failure hook: consume one of the configured failures // before anything else, so the tick outcome maps to `Transient` and // the loop must sleep + retry. @@ -92,7 +92,7 @@ impl BatchPoster for TestMock { .push((batch_index, payload.len())); tx_hashes.push(TxHash::ZERO); } - Ok(tx_hashes) + Ok(SubmitBatchesOutcome::Submitted(tx_hashes)) } async fn observed_submitted_batch_nonces( @@ -175,6 +175,8 @@ async fn submitter_loop_submits_closed_batches_then_exits_on_shutdown() { let shutdown = ShutdownSignal::default(); let config = BatchSubmitterConfig { idle_poll_interval_ms: 5000, + confirmation_depth: 0, + seconds_per_block: 1, }; let submitter = BatchSubmitter::new(path, mock.clone(), config); let handle = submitter @@ -231,6 +233,8 @@ async fn submitter_re_enters_immediately_after_productive_tick() { // Ten seconds — anything above ~2s would be enough to fail if the // immediate-retry cadence regressed to always-sleep. idle_poll_interval_ms: 10_000, + confirmation_depth: 0, + seconds_per_block: 1, }; let submitter = BatchSubmitter::new(path.clone(), mock.clone(), config); let handle = submitter @@ -286,6 +290,8 @@ async fn submitter_recovers_from_transient_poster_error_without_exiting() { // test window. Still long enough that accidentally always-sleeping // would delay the single submission past the assertion. idle_poll_interval_ms: 50, + confirmation_depth: 0, + seconds_per_block: 1, }; let submitter = BatchSubmitter::new(path.clone(), mock.clone(), config); let handle = submitter