From 0ecef48f410da711ffe114870fbe19802d98cc31 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Mon, 26 Oct 2026 21:08:25 -0700 Subject: [PATCH 1/2] orchestrator: recover stalled ticket redemptions Retry eligible failed submissions and replace transactions that exceed a configurable inclusion timeout. Persist signed transactions, acknowledged nonces, and replacement history so recovery can resume after a restart. Reconcile older transactions through finality, repair nonce gaps, and retain ticket liability across reorgs and unresolved attempts. Add recovery settings, a database migration, and tests for shared-wallet and failure scenarios. --- .../testdata/cli/orchestrator-help.golden | 66 +- configs/orchestrator/config.example.toml | 2 + docs/configuration.md | 6 +- docs/orchestrator.md | 4 +- docs/payment-recovery.md | 61 +- eth/contracts.go | 25 +- eth/contracts/README.md | 6 +- eth/key.go | 13 + eth/noncemanager.go | 96 ++ eth/noncemanager_test.go | 120 ++ eth/payment.go | 115 +- eth/rpc.go | 12 + eth/rpc_test.go | 12 + eth/transactionmanager.go | 398 ++++++ eth/transactionmanager_test.go | 364 ++++++ orchestrator/app.go | 71 +- orchestrator/app_test.go | 1 + .../002_redemption_transactions.sql | 41 + orchestrator/redeemer.go | 148 ++- orchestrator/redeemer_db.go | 111 +- orchestrator/redeemer_db_test.go | 54 + orchestrator/redeemer_test.go | 77 ++ orchestrator/redemption_nonce.go | 176 +++ orchestrator/redemption_nonce_test.go | 246 ++++ orchestrator/redemption_recovery.go | 374 ++++++ orchestrator/redemption_recovery_test.go | 1147 +++++++++++++++-- orchestrator/redemption_regression_test.go | 112 -- orchestrator/redemption_store.go | 215 ++- 28 files changed, 3563 insertions(+), 510 deletions(-) create mode 100644 eth/noncemanager.go create mode 100644 eth/noncemanager_test.go create mode 100644 eth/transactionmanager.go create mode 100644 eth/transactionmanager_test.go create mode 100644 orchestrator/migrations/002_redemption_transactions.sql create mode 100644 orchestrator/redeemer_test.go create mode 100644 orchestrator/redemption_nonce.go create mode 100644 orchestrator/redemption_nonce_test.go create mode 100644 orchestrator/redemption_recovery.go delete mode 100644 orchestrator/redemption_regression_test.go diff --git a/cmd/livepeer/testdata/cli/orchestrator-help.golden b/cmd/livepeer/testdata/cli/orchestrator-help.golden index 7b21326..02de235 100644 --- a/cmd/livepeer/testdata/cli/orchestrator-help.golden +++ b/cmd/livepeer/testdata/cli/orchestrator-help.golden @@ -11,38 +11,40 @@ Available Commands: redemptions Inspect recipient redemption state or explicitly retry stored transaction bytes Flags: - -c, --config string Configuration file; empty disables discovery (env: LIVEPEER_ORCHESTRATOR_CONFIG) (default "config.toml") - -n, --network string Network name (custom networks require explicit chain settings) (env: LIVEPEER_ORCHESTRATOR_NETWORK) (default "offchain") - -d, --data-dir string Component data directory; defaults to ~/.lpData// (env: LIVEPEER_ORCHESTRATOR_DATA_DIR) (default "$HOME/.lpData/offchain/orchestrator") - -a, --account string Keystore account address (env: LIVEPEER_ORCHESTRATOR_ACCOUNT) - -o, --orchestrator-address string Registered orchestrator address receiving ticket payments; defaults to the keystore account (env: LIVEPEER_ORCHESTRATOR_ORCHESTRATOR_ADDRESS) - -e, --eth-usd-feed string ETH/USD oracle address, alternative to a fixed wei-per-usd rate (env: LIVEPEER_ORCHESTRATOR_ETH_USD_FEED) - -p, --price-max-age duration Maximum age of the oracle observation (env: LIVEPEER_ORCHESTRATOR_PRICE_MAX_AGE) (default 2h0m0s) - -l, --listen string Public HTTP listener (env: LIVEPEER_ORCHESTRATOR_LISTEN) (default "127.0.0.1:8935") - -m, --metrics-listen string Loopback metrics listener (env: LIVEPEER_ORCHESTRATOR_METRICS_LISTEN) (default "127.0.0.1:8936") - -s, --service-url string Public orchestrator base URL; defaults to http://listen (env: LIVEPEER_ORCHESTRATOR_SERVICE_URL) - -r, --runner-service-url string Runner-facing base URL for callbacks and trickle; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_RUNNER_SERVICE_URL) - --proxy-url-template string Generated proxy URL with {proxy} in a hostname label or final path segment (env: LIVEPEER_ORCHESTRATOR_PROXY_URL_TEMPLATE) - --bootstrap-secret-file string File containing runner bootstrap credential (env: LIVEPEER_ORCHESTRATOR_BOOTSTRAP_SECRET_FILE) - --runner-config string Static runner TOML path (env: LIVEPEER_ORCHESTRATOR_RUNNER_CONFIG) - --redeemer-db string (env: LIVEPEER_ORCHESTRATOR_REDEEMER_DB) - -k, --keystore-file string Encrypted geth redemption account JSON file (env: LIVEPEER_ORCHESTRATOR_KEYSTORE_FILE) - --keystore-password-file string Owner-only file containing the exact keystore password bytes (env: LIVEPEER_ORCHESTRATOR_KEYSTORE_PASSWORD_FILE) - --rpc-url-file string (env: LIVEPEER_ORCHESTRATOR_RPC_URL_FILE) - --chain-id uint (env: LIVEPEER_ORCHESTRATOR_CHAIN_ID) - --redeemer-max-fee-per-gas string Optional maximum redemption fee in wei per gas (env: LIVEPEER_ORCHESTRATOR_REDEEMER_MAX_FEE_PER_GAS) - --controller-address string (env: LIVEPEER_ORCHESTRATOR_CONTROLLER_ADDRESS) - -w, --wei-per-usd string (env: LIVEPEER_ORCHESTRATOR_WEI_PER_USD) - -t, --ticket-face-value string (env: LIVEPEER_ORCHESTRATOR_TICKET_FACE_VALUE) - --ticket-win-prob string (env: LIVEPEER_ORCHESTRATOR_TICKET_WIN_PROB) - --runner-grants strings Private runner IP[:port] or CIDR grants (env: LIVEPEER_ORCHESTRATOR_RUNNER_GRANTS) - --session-proxy-grants strings Private session proxy IP[:port] or CIDR grants (env: LIVEPEER_ORCHESTRATOR_SESSION_PROXY_GRANTS) - --health-grants strings Private static runner health IP[:port] or CIDR grants (env: LIVEPEER_ORCHESTRATOR_HEALTH_GRANTS) - --heartbeat-interval duration Runner heartbeat interval (env: LIVEPEER_ORCHESTRATOR_HEARTBEAT_INTERVAL) (default 5s) - --heartbeat-ttl duration Runner heartbeat expiry (env: LIVEPEER_ORCHESTRATOR_HEARTBEAT_TTL) (default 30s) - --print-config Print audited redacted TOML configuration - -h, --help help for livepeer-orchestrator - -v, --version version for livepeer-orchestrator + -c, --config string Configuration file; empty disables discovery (env: LIVEPEER_ORCHESTRATOR_CONFIG) (default "config.toml") + -n, --network string Network name (custom networks require explicit chain settings) (env: LIVEPEER_ORCHESTRATOR_NETWORK) (default "offchain") + -d, --data-dir string Component data directory; defaults to ~/.lpData// (env: LIVEPEER_ORCHESTRATOR_DATA_DIR) (default "$HOME/.lpData/offchain/orchestrator") + -a, --account string Keystore account address (env: LIVEPEER_ORCHESTRATOR_ACCOUNT) + -o, --orchestrator-address string Registered orchestrator address receiving ticket payments; defaults to the keystore account (env: LIVEPEER_ORCHESTRATOR_ORCHESTRATOR_ADDRESS) + -e, --eth-usd-feed string ETH/USD oracle address, alternative to a fixed wei-per-usd rate (env: LIVEPEER_ORCHESTRATOR_ETH_USD_FEED) + -p, --price-max-age duration Maximum age of the oracle observation (env: LIVEPEER_ORCHESTRATOR_PRICE_MAX_AGE) (default 2h0m0s) + -l, --listen string Public HTTP listener (env: LIVEPEER_ORCHESTRATOR_LISTEN) (default "127.0.0.1:8935") + -m, --metrics-listen string Loopback metrics listener (env: LIVEPEER_ORCHESTRATOR_METRICS_LISTEN) (default "127.0.0.1:8936") + -s, --service-url string Public orchestrator base URL; defaults to http://listen (env: LIVEPEER_ORCHESTRATOR_SERVICE_URL) + -r, --runner-service-url string Runner-facing base URL for callbacks and trickle; defaults to service-url (env: LIVEPEER_ORCHESTRATOR_RUNNER_SERVICE_URL) + --proxy-url-template string Generated proxy URL with {proxy} in a hostname label or final path segment (env: LIVEPEER_ORCHESTRATOR_PROXY_URL_TEMPLATE) + --bootstrap-secret-file string File containing runner bootstrap credential (env: LIVEPEER_ORCHESTRATOR_BOOTSTRAP_SECRET_FILE) + --runner-config string Static runner TOML path (env: LIVEPEER_ORCHESTRATOR_RUNNER_CONFIG) + --redeemer-db string (env: LIVEPEER_ORCHESTRATOR_REDEEMER_DB) + -k, --keystore-file string Encrypted geth redemption account JSON file (env: LIVEPEER_ORCHESTRATOR_KEYSTORE_FILE) + --keystore-password-file string Owner-only file containing the exact keystore password bytes (env: LIVEPEER_ORCHESTRATOR_KEYSTORE_PASSWORD_FILE) + --rpc-url-file string (env: LIVEPEER_ORCHESTRATOR_RPC_URL_FILE) + --chain-id uint (env: LIVEPEER_ORCHESTRATOR_CHAIN_ID) + --redeemer-tx-timeout duration Inclusion timeout for each redemption transaction attempt (env: LIVEPEER_ORCHESTRATOR_REDEEMER_TX_TIMEOUT) (default 5m0s) + --redeemer-max-tx-replacements int Maximum fee replacements per redemption attempt (env: LIVEPEER_ORCHESTRATOR_REDEEMER_MAX_TX_REPLACEMENTS) (default 1) + --redeemer-max-fee-per-gas string Optional maximum redemption fee in wei per gas (env: LIVEPEER_ORCHESTRATOR_REDEEMER_MAX_FEE_PER_GAS) + --controller-address string (env: LIVEPEER_ORCHESTRATOR_CONTROLLER_ADDRESS) + -w, --wei-per-usd string (env: LIVEPEER_ORCHESTRATOR_WEI_PER_USD) + -t, --ticket-face-value string (env: LIVEPEER_ORCHESTRATOR_TICKET_FACE_VALUE) + --ticket-win-prob string (env: LIVEPEER_ORCHESTRATOR_TICKET_WIN_PROB) + --runner-grants strings Private runner IP[:port] or CIDR grants (env: LIVEPEER_ORCHESTRATOR_RUNNER_GRANTS) + --session-proxy-grants strings Private session proxy IP[:port] or CIDR grants (env: LIVEPEER_ORCHESTRATOR_SESSION_PROXY_GRANTS) + --health-grants strings Private static runner health IP[:port] or CIDR grants (env: LIVEPEER_ORCHESTRATOR_HEALTH_GRANTS) + --heartbeat-interval duration Runner heartbeat interval (env: LIVEPEER_ORCHESTRATOR_HEARTBEAT_INTERVAL) (default 5s) + --heartbeat-ttl duration Runner heartbeat expiry (env: LIVEPEER_ORCHESTRATOR_HEARTBEAT_TTL) (default 30s) + --print-config Print audited redacted TOML configuration + -h, --help help for livepeer-orchestrator + -v, --version version for livepeer-orchestrator Use "livepeer-orchestrator [command] --help" for more information about a command. diff --git a/configs/orchestrator/config.example.toml b/configs/orchestrator/config.example.toml index 9bcef84..0f0782f 100644 --- a/configs/orchestrator/config.example.toml +++ b/configs/orchestrator/config.example.toml @@ -29,6 +29,8 @@ HealthGrants = [] # Both files must be owner-only; the password is read as exact bytes (no newline). # Or use LIVEPEER_ORCHESTRATOR_KEYSTORE_PASSWORD and omit KeystorePasswordFile. # RPCURLFile = "/run/secrets/livepeer-payment-rpc-url" +# RedeemerTxTimeout = "5m" # Inclusion timeout per transaction attempt. +# RedeemerMaxTxReplacements = 1 # Same-nonce fee replacements before retrying the ticket. # RedeemerMaxFeePerGas = "1000000000" # Optional ceiling, wei per gas # ChainID = 42161 # Controller = "0xD8E8328501E9645d16Cf49539efC04f734606ee4" # Arbitrum default. diff --git a/docs/configuration.md b/docs/configuration.md index 58467a7..1300a41 100644 --- a/docs/configuration.md +++ b/docs/configuration.md @@ -132,9 +132,9 @@ a new chain invocation. The signer account is the payer, supplying the TicketBroker deposit and reserve. The orchestrator's keystore account signs redemptions and pays gas. Ticket payments credit `OrchestratorAddress`, which defaults to that account. The chain account -must match the keystore account when signing. Keep a redemption key exclusive to -its orchestrator worker; concurrent CLI or other use can conflict with nonce -tracking. +must match the keystore account when signing. Multiple orchestrator processes +may share a redemption key. Failed submissions are retried automatically. +See [transaction recovery](payment-recovery.md). ### Prepare a keystore diff --git a/docs/orchestrator.md b/docs/orchestrator.md index de61288..c379338 100644 --- a/docs/orchestrator.md +++ b/docs/orchestrator.md @@ -130,8 +130,8 @@ every hour; `PriceMaxAge` must be positive and defaults to two hours. Startup requires a valid observation. Stale rates suspend new sessions; existing sessions retain their initial price. -`RedeemerMaxFeePerGas` optionally caps redemption fees in wei per gas. The -cap is per gas, not a total spending budget. Winning tickets wait until payment +See [transaction recovery](payment-recovery.md#orchestrator-transaction-recovery) +for timeouts, replacements and fee settings. Winning tickets wait until payment parameters expire before redemption exposes their randomness on-chain. Only a successful, canonical, finalized receipt settles their liability. diff --git a/docs/payment-recovery.md b/docs/payment-recovery.md index 63dc252..8bb4c6e 100644 --- a/docs/payment-recovery.md +++ b/docs/payment-recovery.md @@ -56,18 +56,37 @@ use these states: | State | Meaning and recovery | | --- | --- | | Queued (no attempt) | Wait for parameter expiry. Preparation failures leave the ticket eligible for another attempt. | -| `prepared` | Signed bytes, nonce and locally computed hash are durable; the worker resumes the first send after restart. | -| `broadcast` | A send may have reached RPC. Reconcile receipts automatically; an operator may explicitly resend identical bytes. | -| `submitted` | RPC acknowledged the hash. Keep checking receipts; explicit identical-byte retry is available if necessary. | -| `confirmed` | Receipt is successful, at or below the finalized head, and matches the canonical block hash. Liability is settled. | -| `reverted` | A canonical finalized receipt failed. Retain the audit row; do not resubmit automatically. | -| `expired` | Ticket creation round is outside the contract validity window. Retain the audit row. | - -An uncertain broadcast blocks preparation of a new transaction -for this worker, preventing accidental nonce reuse. The redemption key must be -exclusive to this worker; do not submit concurrent CLI transactions from it. -Only a canonical finalized receipt marks a ticket redeemed. A pending receipt, -reorg, unavailable finalized head or RPC failure retains uncertainty and liability. +| `prepared` | Signed bytes, nonce and hash are durable before sending; restart resumes the saved transaction. | +| `broadcast` | Send in progress or interrupted before recording its outcome; restart resumes the saved transaction. | +| `submitted` | RPC acknowledged the hash; monitor inclusion and replace on timeout. | +| `retryable` | Submission, waiting or replacement failed; retry the ticket automatically. Older hashes remain under receipt reconciliation. | +| `mined` | Canonical inclusion observed; wait for finalization without fee replacement. A reorg resumes recovery. | +| `awaiting_finality` | Latest state says the ticket is used; pause submission until finalized state confirms use or a reorg removes it. | +| `confirmed` / `used` | Successful canonical finalized receipt or finalized ticket-use observation settles liability. | +| `reverted` / `invalid` | Finalized transaction failure or an invalid ticket; stop submitting and reconcile unresolved hashes. | +| `expired` | Ticket creation round is outside its validity window; reconcile unresolved hashes. | +| `superseded` | Finalized inclusion consumed the same sender/nonce, or finalized ticket settlement made this transaction unnecessary. | + +Only canonical finalized state settles a ticket. RPC failures, missing receipts, +unfinalized inclusions and reorgs retain uncertainty and liability. Each ticket +counts once even when it has several signed transactions. Previously exposed +recipient randomness stays closed to new payments throughout retries. + +`RedeemerTxTimeout` (`--redeemer-tx-timeout`) defaults to five minutes per +inclusion attempt. `RedeemerMaxTxReplacements` (`--redeemer-max-tx-replacements`) +defaults to one; zero disables replacements. Replacements retain the nonce and +calldata and raise both fee caps by geth's default 10% threshold, rounding up +with a strict increase. +Geth estimates initial fees; `RedeemerMaxFeePerGas` caps initial and replacement +fees in wei per gas. Finalization waits do not consume the inclusion timeout. + +Restart restores deadlines, replacement budgets and nonce reservations for +acknowledged sends. Rejected submissions do not reserve a nonce. A rotated key can +resend saved bytes and reconcile receipts, but cannot replace the previous account's transactions. + +Migration 002 preserves existing transactions. Its rollback refuses to discard +transactions or metadata that the old schema cannot represent. Retain the upgraded +database for a compatible binary when rollback is refused. Inspect status without a key or RPC connection: @@ -75,8 +94,8 @@ Inspect status without a key or RPC connection: livepeer orchestrator redemptions ``` -After checking the hash on the configured chain, explicitly retry the stored -transaction if it is absent or needs rebroadcast: +Automatic recovery normally handles rebroadcast. To explicitly resend the +current retryable transaction using its saved bytes and fees: ```sh livepeer orchestrator redemptions \ @@ -91,9 +110,8 @@ pass the same `--config`, or `--network` and `--data-dir`, as the service. `--redeemer-db` selects a different database. Recovery reads `RPCURLFile` and `ChainID` for retries; `--rpc-url-file` and `--chain-id` override them. It does not unlock the account or read unrelated service credentials. RPC endpoints use normal -system TLS trust. This operation uses the exact saved signed bytes, nonce and -hash; it cannot create a replacement -or change fees. A retry error retains the hash and uncertain state. +system TLS trust. Historical hashes remain inspectable; manual retry cannot +reopen a terminal ticket or displace a newer active transaction. In-memory control state is bounded: payment sessions have a 100,000-entry quota and 24-hour inactivity retention; ticket replay guards have a 1,000,000-nonce @@ -103,12 +121,3 @@ a backward chain observation cannot revive parameters after their replay guards are removed. Auth-token refresh never resets nonce tracking. Winning tickets and uncertain transactions are retained for audit/reconciliation, so back up and monitor database disk use independently of these control-state quotas. - -Redemption planning uses geth's dynamic-fee selection and requires a base fee -in the RPC header; a missing base fee leaves the ticket queued with an error. -`RedeemerMaxFeePerGas` optionally caps the fee in wei per gas; a plan above -that ceiling remains queued without signing or broadcasting. Prepared and -submitted identities reserve their nonces, including after restart. Keep the -redemption key exclusive to this database/worker; local nonce tracking does not -coordinate another process using the same key. Explicit retry always resends -the stored transaction without changing its fees. diff --git a/eth/contracts.go b/eth/contracts.go index f0d7b23..df35331 100644 --- a/eth/contracts.go +++ b/eth/contracts.go @@ -255,6 +255,12 @@ func (c *Contracts) PrepareAndStore(ctx context.Context, plan TransactionPlan, k } func (c *Contracts) prepare(ctx context.Context, plan TransactionPlan, key *Key, chainID *big.Int, save func(SignedTransaction) error) (SignedTransaction, error) { + return c.prepareAtNonce(ctx, plan, key, chainID, save, nil) +} + +// prepareAtNonce lets the ported redemption backend own nonce allocation through +// submission. The CLI's prepare-and-reserve behavior remains unchanged. +func (c *Contracts) prepareAtNonce(ctx context.Context, plan TransactionPlan, key *Key, chainID *big.Int, save func(SignedTransaction) error, explicit *uint64) (SignedTransaction, error) { if key == nil || key.Address() != plan.From || chainID == nil || chainID.Sign() <= 0 { return SignedTransaction{}, errors.New("transaction account does not match signing key") } @@ -297,12 +303,15 @@ func (c *Contracts) prepare(ctx context.Context, plan TransactionPlan, key *Key, state := c.nonceState(plan.From) state.Lock() defer state.Unlock() - nonce, err := c.RPC.ethereum.PendingNonceAt(ctx, plan.From) - if err != nil { - return SignedTransaction{}, safeRPCError(err) - } - if nonce < state.next { - nonce = state.next + var nonce uint64 + if explicit != nil { + nonce = *explicit + } else { + nonce, err = c.RPC.ethereum.PendingNonceAt(ctx, plan.From) + if err != nil { + return SignedTransaction{}, safeRPCError(err) + } + nonce = max(nonce, state.next) } if nonce == math.MaxUint64 { return SignedTransaction{}, errors.New("transaction account nonce exhausted") @@ -325,7 +334,9 @@ func (c *Contracts) prepare(ctx context.Context, plan TransactionPlan, key *Key, } // Once saved or returned to a caller, this identity can be broadcast even // if its outcome is uncertain. Never reuse its nonce in this client. - state.next = nonce + 1 + if explicit == nil { + state.next = nonce + 1 + } return prepared, nil } diff --git a/eth/contracts/README.md b/eth/contracts/README.md index db005dd..0a5a2c4 100644 --- a/eth/contracts/README.md +++ b/eth/contracts/README.md @@ -1,7 +1,7 @@ # Contract bindings The ten generated Go files in this directory and `chainlink/` are copied -unchanged from `livepeer/go-livepeer` commit +unchanged from source commit `bd645a09266833fb859053445d9ac85846330756`: - `controller.go` @@ -17,7 +17,7 @@ unchanged from `livepeer/go-livepeer` commit Their filtered source history is a parent of the binding import commit. It preserves the authors, committers, dates, and messages, with `Original-Commit` -trailers linking each retained commit to `livepeer/go-livepeer`. Contributors +trailers recording each retained commit's original identity. Contributors include Eric Tang, Yondon Fu, Nico Vergauwen, Rafał Leszko, Rick Staa, and Victor Elias. The original MIT notice is in [`LICENSE`](../../LICENSE). @@ -27,7 +27,7 @@ copy these files from the selected committed revision, review its contract/API changes, and update this provenance. For example, from a checkout of this repo: ```sh -git -C /path/to/go-livepeer show REVISION:eth/contracts/bondingManager.go > eth/contracts/bondingManager.go +git -C /path/to/protocol-source show REVISION:eth/contracts/bondingManager.go > eth/contracts/bondingManager.go ``` Application policy lives in the parent `eth` package: canonical snapshots, diff --git a/eth/key.go b/eth/key.go index ef6974b..bc84516 100644 --- a/eth/key.go +++ b/eth/key.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "math/big" "os" "path/filepath" "strings" @@ -15,6 +16,7 @@ import ( "github.com/ethereum/go-ethereum/accounts/keystore" ethcommon "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" ) @@ -22,6 +24,17 @@ import ( // boundary and is not included in command output or errors. type Key struct{ private *ecdsa.PrivateKey } +// SignTx adapts the local key to the ported go-livepeer transaction signer. +func (k *Key) SignTx(tx *types.Transaction) (*types.Transaction, error) { + return k.SignTxForChain(tx, tx.ChainId()) +} + +// SignTxForChain retains upstream accountManager's chain binding when an +// unsigned legacy replacement does not yet encode its original chain ID. +func (k *Key) SignTxForChain(tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { + return types.SignTx(tx, types.LatestSignerForChainID(chainID), k.private) +} + // SelectKeystore selects metadata without unlocking a key. An explicit file // bypasses directory discovery; its address is checked again after decryption. func SelectKeystore(dir, explicit string, account *ethcommon.Address) (string, ethcommon.Address, error) { diff --git a/eth/noncemanager.go b/eth/noncemanager.go new file mode 100644 index 0000000..a711718 --- /dev/null +++ b/eth/noncemanager.go @@ -0,0 +1,96 @@ +// Ported from livepeer/go-livepeer at 0ff08af1bb4cb268f428d619cc968b5eaee0d093. +// Original implementation by Yondon Fu and Rafal Leszko; MIT notice in LICENSE. +package eth + +import ( + "context" + "sync" + + ethcommon "github.com/ethereum/go-ethereum/common" +) + +// RemoteNonceReader is an interface that describes an object +// capable of reading transaction nonces for ETH address from a remote source +type RemoteNonceReader interface { + PendingNonceAt(ctx context.Context, addr ethcommon.Address) (uint64, error) +} + +type nonceLock struct { + nonce uint64 + mu sync.Mutex +} + +// NonceManager manages transaction nonces for multiple ETH addresses +type NonceManager struct { + nonces map[ethcommon.Address]*nonceLock + mu sync.Mutex + + remoteReader RemoteNonceReader +} + +// NewNonceManager creates an instance of a NonceManager +func NewNonceManager(remoteReader RemoteNonceReader) *NonceManager { + return &NonceManager{ + nonces: make(map[ethcommon.Address]*nonceLock), + remoteReader: remoteReader, + } +} + +// Lock locks the provided address. The caller should always call Lock before +// calling Next or Update +func (m *NonceManager) Lock(addr ethcommon.Address) { + m.getNonceLock(addr).mu.Lock() +} + +// Unlock unlocks the provided address. The caller should always call Unlock +// after finishing calls to Next or Update +func (m *NonceManager) Unlock(addr ethcommon.Address) { + m.getNonceLock(addr).mu.Unlock() +} + +// Next returns the next transaction nonce to be used for the provided address +func (m *NonceManager) Next(addr ethcommon.Address) (uint64, error) { + return m.NextContext(context.Background(), addr) +} + +// NextContext adapts upstream Next to the service/RPC cancellation boundary. +func (m *NonceManager) NextContext(ctx context.Context, addr ethcommon.Address) (uint64, error) { + nonceLock := m.getNonceLock(addr) + localNonce := nonceLock.nonce + + remoteNonce, err := m.remoteReader.PendingNonceAt(ctx, addr) + if err != nil { + return 0, err + } + + // If remote nonce > local nonce, another client was likely used + // to submit transactions such that the local nonce does not capture + // transactions submitted by other clients + return max(remoteNonce, localNonce), nil +} + +// Update uses the last nonce for the provided address to update the next transaction nonce +func (m *NonceManager) Update(addr ethcommon.Address, lastNonce uint64) { + nonceLock := m.getNonceLock(addr) + nonceLock.nonce = lastNonce + 1 +} + +func (m *NonceManager) getNonceLock(addr ethcommon.Address) *nonceLock { + m.mu.Lock() + defer m.mu.Unlock() + + if _, ok := m.nonces[addr]; !ok { + m.nonces[addr] = new(nonceLock) + } + + return m.nonces[addr] +} + +// Restore includes acknowledged durable sends when a redemption worker restarts. +// It cannot lower the upstream cursor when an older transaction is rebroadcast. +func (m *NonceManager) Restore(addr ethcommon.Address, next uint64) { + m.Lock(addr) + defer m.Unlock(addr) + state := m.getNonceLock(addr) + state.nonce = max(state.nonce, next) +} diff --git a/eth/noncemanager_test.go b/eth/noncemanager_test.go new file mode 100644 index 0000000..3cc731f --- /dev/null +++ b/eth/noncemanager_test.go @@ -0,0 +1,120 @@ +// Ported from livepeer/go-livepeer at 0ff08af1bb4cb268f428d619cc968b5eaee0d093. +// Original implementation by Yondon Fu and Rafal Leszko; MIT notice in LICENSE. +package eth + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "sync" + "sync/atomic" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/stretchr/testify/require" +) + +// The upstream tests use testify/mock; this fixed reader avoids adding its +// separate objx dependency. The table retains the upstream nonce assertions. +type mockRemoteNonceReader struct { + nonce uint64 + err error +} + +func (m *mockRemoteNonceReader) PendingNonceAt(context.Context, common.Address) (uint64, error) { + return m.nonce, m.err +} + +func TestNonceManagerNextAndUpdate(t *testing.T) { + for _, tc := range []struct { + name string + remote uint64 + last uint64 + update bool + readBeforeUpdate bool + want uint64 + err error + }{ + {name: "RPC error", err: errors.New("PendingNonceAt error")}, + {name: "zero"}, + {name: "increment local", update: true, readBeforeUpdate: true, want: 1}, + {name: "remote ahead", remote: 10, want: 10}, + {name: "last equals local", update: true, want: 1}, + {name: "last ahead of local", update: true, last: 10, want: 11}, + } { + t.Run(tc.name, func(t *testing.T) { + nm := NewNonceManager(&mockRemoteNonceReader{nonce: tc.remote, err: tc.err}) + addr := randomNonceAddress() + if tc.update { + // Preserve both upstream paths: Update after Next, and Update + // before any nonce has been read for the account. + if tc.readBeforeUpdate { + nonce, err := nm.Next(addr) + require.NoError(t, err) + require.Zero(t, nonce) + } + nm.Update(addr, tc.last) + } + nonce, err := nm.Next(addr) + require.ErrorIs(t, err, tc.err) + require.Equal(t, tc.want, nonce) + }) + } +} + +func TestNonceManagerConcurrent(t *testing.T) { + for _, shared := range []bool{false, true} { + t.Run(fmt.Sprintf("shared=%t", shared), func(t *testing.T) { + nm := NewNonceManager(&mockRemoteNonceReader{}) + var wg sync.WaitGroup + var errCount atomic.Uint64 + var usedMu sync.Mutex + used := map[uint64]bool{} + var addrs [50]common.Address + sharedAddr := randomNonceAddress() + for i := range addrs { + addrs[i] = sharedAddr + if !shared { + addrs[i] = randomNonceAddress() + } + wg.Add(1) + go func(addr common.Address) { + defer wg.Done() + nm.Lock(addr) + defer nm.Unlock(addr) + nonce, err := nm.Next(addr) + if err != nil { + errCount.Add(1) + return + } + if shared { + usedMu.Lock() + used[nonce] = true + usedMu.Unlock() + } + nm.Update(addr, nonce) + }(addrs[i]) + } + wg.Wait() + require.Zero(t, errCount.Load()) + for _, addr := range addrs { + nonce, err := nm.Next(addr) + require.NoError(t, err) + want := uint64(1) + if shared { + want = 50 + } + require.Equal(t, want, nonce) + } + if shared { + require.Len(t, used, 50) + for i := uint64(0); i < 50; i++ { + require.True(t, used[i]) + } + } + }) + } +} + +func randomNonceAddress() common.Address { var a common.Address; _, _ = rand.Read(a[:]); return a } diff --git a/eth/payment.go b/eth/payment.go index 082f57c..196db67 100644 --- a/eth/payment.go +++ b/eth/payment.go @@ -171,39 +171,118 @@ type RedeemTicket struct { } func (c PaymentChain) PrepareRedemption(ctx context.Context, redeemer *Key, chainID *big.Int, t RedeemTicket) (SignedTransaction, error) { - return c.prepareRedemption(ctx, redeemer, chainID, t, nil) -} - -// PrepareRedemptionAndStore reserves the nonce only after the caller saves the -// signed redemption identity, before any possible broadcast. -func (c PaymentChain) PrepareRedemptionAndStore(ctx context.Context, redeemer *Key, chainID *big.Int, t RedeemTicket, save func(SignedTransaction) error) (SignedTransaction, error) { - if save == nil { - return SignedTransaction{}, errors.New("transaction persistence is required") + plan, err := c.redemptionPlan(ctx, redeemer, t) + if err != nil { + return SignedTransaction{}, err } - return c.prepareRedemption(ctx, redeemer, chainID, t, save) + return c.Contracts.Prepare(ctx, plan, redeemer, chainID) } -func (c PaymentChain) prepareRedemption(ctx context.Context, redeemer *Key, chainID *big.Int, t RedeemTicket, save func(SignedTransaction) error) (SignedTransaction, error) { +func (c PaymentChain) redemptionPlan(ctx context.Context, redeemer *Key, t RedeemTicket) (TransactionPlan, error) { if redeemer == nil || t.Recipient == (ethcommon.Address{}) || t.FaceValue == nil || t.WinProb == nil || t.RecipientRand == nil { - return SignedTransaction{}, errors.New("incomplete redemption configuration or ticket") + return TransactionPlan{}, errors.New("incomplete redemption configuration or ticket") } address, err := c.Contracts.Resolve(ctx, "ticketBroker") if err != nil { - return SignedTransaction{}, err + return TransactionPlan{}, err } coreTicket := contracts.MTicketBrokerCoreTicket{Recipient: t.Recipient, Sender: t.Sender, FaceValue: t.FaceValue, WinProb: t.WinProb, SenderNonce: new(big.Int).SetUint64(uint64(t.SenderNonce)), RecipientRandHash: t.RecipientRandHash, AuxData: t.AuxData} data, err := c.Contracts.Pack("ticketBroker", "redeemWinningTicket", coreTicket, t.Signature, t.RecipientRand) if err != nil { - return SignedTransaction{}, err + return TransactionPlan{}, err } - plan, err := c.Contracts.PlanTransaction(ctx, redeemer.Address(), address, data, big.NewInt(0)) + return c.Contracts.PlanTransaction(ctx, redeemer.Address(), address, data, big.NewInt(0)) +} + +// SubmitRedemption ports go-livepeer/eth/backend.go's Next/send/Update sequence. +// Persistence happens before send, but only an acknowledged send advances Next. +func (c PaymentChain) SubmitRedemption(ctx context.Context, nonces *NonceManager, manager *TransactionManager, redeemer *Key, chainID *big.Int, t RedeemTicket, save func(SignedTransaction) error) error { + nonces.Lock(redeemer.Address()) + defer nonces.Unlock(redeemer.Address()) + plan, err := c.redemptionPlan(ctx, redeemer, t) if err != nil { - return SignedTransaction{}, err + return err + } + nonce, err := nonces.NextContext(ctx, redeemer.Address()) + if err != nil { + return err } - if save != nil { - return c.Contracts.PrepareAndStore(ctx, plan, redeemer, chainID, save) + prepared, err := c.Contracts.prepareAtNonce(ctx, plan, redeemer, chainID, save, &nonce) + if err != nil { + return err } - return c.Contracts.Prepare(ctx, plan, redeemer, chainID) + var tx types.Transaction + if err := tx.UnmarshalBinary(prepared.Raw); err != nil { + return err + } + if err := manager.SendTransaction(ctx, &tx); err != nil { + return err + } + nonces.Update(redeemer.Address(), nonce) + return nil +} + +// IsUsedTicket is ported from go-livepeer/eth/client_ticketbroker.go. The block +// argument also supports the finalized observation needed for node settlement. +func (c PaymentChain) IsUsedTicket(ctx context.Context, hash ethcommon.Hash, block string) (bool, error) { + var ref any = block + if block == "finalized" { + header, err := c.Contracts.RPC.header(ctx, "finalized") + if err != nil { + return false, err + } + if header.Hash == (ethcommon.Hash{}) || header.Number == nil { + return false, errors.New("invalid finalized header") + } + ref = map[string]any{"blockHash": header.Hash, "requireCanonical": true} + } + address, err := c.Contracts.ResolveAt(ctx, ref, "ticketBroker") + if err != nil { + return false, err + } + broker, err := contracts.NewTicketBrokerCaller(address, c.Contracts.callerAt(ref)) + if err != nil { + return false, err + } + return broker.UsedTickets(&bind.CallOpts{Context: ctx}, hash) +} + +// FinalizedNonce pins account state to the same canonical finalized reference +// used for ticket-use settlement. A missing/malformed count is not evidence. +func (c PaymentChain) FinalizedNonce(ctx context.Context, address ethcommon.Address) (uint64, error) { + header, err := c.Contracts.RPC.header(ctx, "finalized") + if err != nil { + return 0, err + } + if header.Hash == (ethcommon.Hash{}) || header.Number == nil || !(*big.Int)(header.Number).IsUint64() { + return 0, errors.New("invalid finalized header") + } + var nonce *hexutil.Uint64 + ref := map[string]any{"blockHash": header.Hash, "requireCanonical": true} + if err := c.Contracts.RPC.ethereum.Client().CallContext(ctx, &nonce, "eth_getTransactionCount", address, ref); err != nil { + return 0, safeRPCError(err) + } + if nonce == nil { + return 0, errors.New("invalid finalized transaction count") + } + return uint64(*nonce), nil +} + +// InclusionReceipt retains node's canonical-block check without treating an +// unfinalized inclusion as settled. The ported manager only waits for inclusion. +func (c PaymentChain) InclusionReceipt(ctx context.Context, hash ethcommon.Hash) (*types.Receipt, error) { + receipt, err := c.Contracts.RPC.TransactionReceipt(ctx, hash) + if err != nil { + return nil, err + } + header, err := c.Contracts.RPC.header(ctx, hexutil.EncodeBig(receipt.BlockNumber)) + if err != nil { + return nil, err + } + if header.Hash != receipt.BlockHash { + return nil, ethereum.NotFound + } + return receipt, nil } func (c PaymentChain) Redeem(ctx context.Context, redeemer *Key, chainID *big.Int, t RedeemTicket) (ethcommon.Hash, error) { diff --git a/eth/rpc.go b/eth/rpc.go index ced78e1..117060f 100644 --- a/eth/rpc.go +++ b/eth/rpc.go @@ -105,11 +105,23 @@ func safeRPCError(err error) error { } var rpcError gethrpc.Error if errors.As(err, &rpcError) { + // Preserve only upstream's fixed terminal ticket classification. Arbitrary + // provider text (including endpoints or calldata) must remain redacted. + if strings.Contains(rpcError.Error(), "ticket creationRound does not have a block hash") { + return ErrTicketCreationRound + } return fmt.Errorf("ethereum RPC error %d", rpcError.ErrorCode()) } return errors.New("ethereum RPC request failed") } +var ErrTicketCreationRound = errors.New("ticket creationRound does not have a block hash") + +func (r *RPC) TransactionByHash(ctx context.Context, hash ethcommon.Hash) (*types.Transaction, bool, error) { + tx, pending, err := r.ethereum.TransactionByHash(ctx, hash) + return tx, pending, safeRPCError(err) +} + func ValidAddress(address string) bool { if len(address) != 42 || !strings.HasPrefix(address, "0x") { return false diff --git a/eth/rpc_test.go b/eth/rpc_test.go index a7b35df..d9f554d 100644 --- a/eth/rpc_test.go +++ b/eth/rpc_test.go @@ -94,3 +94,15 @@ func testRPC(t *testing.T, reply func(string, []json.RawMessage) any) *RPC { t.Cleanup(rpc.Close) return rpc } + +type ticketRevertRPCError struct{ message string } + +func (e ticketRevertRPCError) Error() string { return e.message } +func (e ticketRevertRPCError) ErrorCode() int { return -32000 } + +func TestTerminalTicketErrorRetainsOnlyKnownClassification(t *testing.T) { + original := ticketRevertRPCError{message: "secret https://user:password@rpc.example: ticket creationRound does not have a block hash; private calldata"} + require.ErrorIs(t, safeRPCError(original), ErrTicketCreationRound) + require.Equal(t, "ticket creationRound does not have a block hash", safeRPCError(original).Error()) + require.Equal(t, "ethereum RPC error -32000", safeRPCError(ticketRevertRPCError{message: "nonce too low https://user:password@rpc.example"}).Error()) +} diff --git a/eth/transactionmanager.go b/eth/transactionmanager.go new file mode 100644 index 0000000..d603db4 --- /dev/null +++ b/eth/transactionmanager.go @@ -0,0 +1,398 @@ +// Ported from livepeer/go-livepeer at 0ff08af1bb4cb268f428d619cc968b5eaee0d093. +// Original implementation by Yondon Fu and Rafal Leszko; MIT notice in LICENSE. +package eth + +import ( + "context" + "errors" + "fmt" + "math/big" + "sync" + "time" + + ethereum "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +// Geth defaults to txpool.pricebump=10. Its RPC client only submits signed +// transactions; it does not create replacements. Keep upstream replacement +// control flow, using geth's default threshold instead of its extra 1% margin. +const priceBump uint64 = 10 + +var ErrReplacingMinedTx = errors.New("cannot replace a mined transaction") + +// Transaction lookup proves inclusion of this hash, not a finalized receipt. +// Keep the identity on the completion result so other attempts stay unresolved. +type includedTransaction struct{ hash ethcommon.Hash } + +func (e includedTransaction) Error() string { return ErrReplacingMinedTx.Error() } +func (e includedTransaction) Unwrap() error { return ErrReplacingMinedTx } + +// A failed live receipt check cannot establish that an inclusion timed out. +// Durable callers retain the monitoring cursor and try the check again. +var ErrReceiptUnavailable = errors.New("transaction inclusion could not be checked") + +type feeCeiling interface{ MaxGasPrice() *big.Int } + +// TransactionRecovery is the persisted upstream inclusion/replacement cursor. +type TransactionRecovery struct { + OriginHash ethcommon.Hash + Transaction *types.Transaction + Hashes []ethcommon.Hash + SentAt time.Time + Replacements int +} + +type transactionSenderReader interface { + ethereum.TransactionSender + TransactionByHash(context.Context, ethcommon.Hash) (*types.Transaction, bool, error) + TransactionReceipt(context.Context, ethcommon.Hash) (*types.Receipt, error) + // Required for bind.DeployBackend argument in bind.WaitMined + CodeAt(context.Context, ethcommon.Address, *big.Int) ([]byte, error) +} + +type transactionSigner interface { + SignTx(tx *types.Transaction) (*types.Transaction, error) +} + +type TransactionManager struct { + txTimeout time.Duration + maxReplacements int + + queue []*types.Transaction + + eth transactionSenderReader + gpm feeCeiling + sig transactionSigner + + cond *sync.Cond + + quit chan struct{} + ctx context.Context + cancel context.CancelFunc + stopOnce sync.Once + // Persistence adapters run before any replacement can reach the network. + Restore func(context.Context, *types.Transaction) (TransactionRecovery, error) + BeforeReplacement func(context.Context, *types.Transaction, *types.Transaction) error + Completed func(context.Context, *transactionReceipt) + current TransactionRecovery +} + +type transactionReceipt struct { + originTxHash ethcommon.Hash + types.Receipt + err error +} + +func NewTransactionManager(eth transactionSenderReader, gpm feeCeiling, signer transactionSigner, txTimeout time.Duration, maxReplacements int) *TransactionManager { + ctx, cancel := context.WithCancel(context.Background()) + return &TransactionManager{ + ctx: ctx, cancel: cancel, + cond: sync.NewCond(&sync.Mutex{}), + txTimeout: txTimeout, + maxReplacements: maxReplacements, + eth: eth, + gpm: gpm, + sig: signer, + quit: make(chan struct{}), + } +} + +func (tm *TransactionManager) SendTransaction(ctx context.Context, tx *types.Transaction) error { + sendErr := tm.eth.SendTransaction(ctx, tx) + + if sendErr != nil { + return sendErr + } + + tm.Track(tx) + return nil +} + +func (tm *TransactionManager) Start() { + tm.checkTxLoop() +} + +func (tm *TransactionManager) Stop() { + tm.stopOnce.Do(func() { + tm.cancel() + tm.cond.L.Lock() + close(tm.quit) + tm.cond.Broadcast() + tm.cond.L.Unlock() + }) +} + +// Track resumes an identity already durably submitted, without broadcasting. +func (tm *TransactionManager) Track(tx *types.Transaction) { + tm.cond.L.Lock() + tm.queue = append(tm.queue, tx) + tm.cond.L.Unlock() + tm.cond.Signal() +} + +func (tm *TransactionManager) Context() context.Context { + return tm.ctx +} + +func (tm *TransactionManager) wait(tx *types.Transaction) (*types.Receipt, error) { + ctx, cancel := context.WithTimeout(tm.Context(), tm.txTimeout) + defer cancel() + + // Resume the original deadline after restart, rather than buying a new timeout. + if !tm.current.SentAt.IsZero() { + cancel() + ctx, cancel = context.WithDeadline(tm.Context(), tm.current.SentAt.Add(tm.txTimeout)) + defer cancel() + } + reader := receiptAttempts{transactionSenderReader: tm.eth, hashes: tm.current.Hashes} + receipt, err := bind.WaitMined(ctx, reader, tx) + if errors.Is(err, context.DeadlineExceeded) && tm.Context().Err() == nil { + // Inclusion deadlines survive restart; canonical inclusion does too. Use + // a live context before replacing or reopening a ticket's retry queue. + found, lookupErr := reader.TransactionReceipt(tm.Context(), tx.Hash()) + if lookupErr == nil { + return found, nil + } + if !errors.Is(lookupErr, ethereum.NotFound) { + return nil, fmt.Errorf("%w: %w", ErrReceiptUnavailable, lookupErr) + } + // Receipt endpoints can lag transaction lookup. Preserve upstream's + // mined guard even when no replacements remain, for every saved hash. + hashes := tm.current.Hashes + if len(hashes) == 0 { + hashes = []ethcommon.Hash{tx.Hash()} + } + var inclusionErr error + for _, hash := range hashes { + _, pending, lookupErr := tm.eth.TransactionByHash(tm.Context(), hash) + if lookupErr == nil && !pending { + return nil, includedTransaction{hash} + } + if lookupErr != nil && !errors.Is(lookupErr, ethereum.NotFound) { + inclusionErr = errors.Join(inclusionErr, lookupErr) + } + } + if inclusionErr != nil { + return nil, fmt.Errorf("%w: %w", ErrReceiptUnavailable, inclusionErr) + } + } + return receipt, err +} + +func (tm *TransactionManager) replace(tx *types.Transaction) (*types.Transaction, error) { + _, pending, err := tm.eth.TransactionByHash(tm.Context(), tx.Hash()) + // Only return here if the error is not related to the tx not being found + // Presumably the provided tx was already broadcasted at some point, so even if for some reason the + // node being used cannot find it, the originally broadcasted tx is still valid and might be sitting somewhere + if err != nil && !errors.Is(err, ethereum.NotFound) { + return nil, err + } + // If tx was found + // If `pending` is false, the tx was mined and included in a block + if err == nil && !pending { + return nil, includedTransaction{tx.Hash()} + } + + // A restarted node may have rotated keys. Stored identities retain their + // original sender; never sign a replacement using a different account. + if signer, ok := tm.sig.(interface{ Address() ethcommon.Address }); ok { + from, err := types.Sender(types.LatestSignerForChainID(tx.ChainId()), tx) + if err != nil || from != signer.Address() { + return nil, errors.New("replacement transaction account mismatch") + } + } + newRawTx := newReplacementTx(tx) + + // Bump gas price exceeds max gas price, return early + max := tm.gpm.MaxGasPrice() + // Node's ceiling bounds the actual fee cap, including EIP-1559 worst case. + newGasPrice := newRawTx.GasFeeCap() + if max != nil && newGasPrice.Cmp(max) > 0 { + return nil, fmt.Errorf("replacement gas price exceeds max gas price suggested=%v max=%v", newGasPrice, max) + } + + var newSignedTx *types.Transaction + // Upstream accountManager already knows its chain. Node's raw Key needs + // that binding explicitly for unsigned legacy replacements (chain ID zero). + if signer, ok := tm.sig.(interface { + SignTxForChain(*types.Transaction, *big.Int) (*types.Transaction, error) + }); ok { + newSignedTx, err = signer.SignTxForChain(newRawTx, tx.ChainId()) + } else { + newSignedTx, err = tm.sig.SignTx(newRawTx) + } + if err != nil { + return nil, err + } + + if tm.BeforeReplacement != nil { + if err := tm.BeforeReplacement(tm.Context(), tx, newSignedTx); err != nil { + return nil, err + } + } + sendErr := tm.eth.SendTransaction(tm.Context(), newSignedTx) + if sendErr == nil { + tm.current.SentAt = time.Now() + tm.current.Hashes = append(tm.current.Hashes, newSignedTx.Hash()) + } + + return newSignedTx, sendErr +} + +func (tm *TransactionManager) checkTxLoop() { + for { + tm.cond.L.Lock() + for len(tm.queue) == 0 { + select { + case <-tm.quit: + tm.cond.L.Unlock() + return + default: + } + tm.cond.Wait() + } + // Stop must also leave already queued jobs for durable restart recovery. + select { + case <-tm.quit: + tm.cond.L.Unlock() + return + default: + } + + tx := tm.queue[0] + tm.queue = tm.queue[1:] + tm.cond.L.Unlock() + + originHash := tx.Hash() + tm.current = TransactionRecovery{} + if tm.Restore != nil { + restored, err := tm.Restore(tm.Context(), tx) + if restored.OriginHash != (ethcommon.Hash{}) { + originHash = restored.OriginHash + } + if err != nil { + tm.deliver(&transactionReceipt{originTxHash: originHash, err: err}) + continue + } + tm.current = restored + if restored.Transaction != nil { + tx = restored.Transaction + } + } + + var txReceipt types.Receipt + + receipt, err := tm.wait(tx) + + // context.DeadlineExceeded indicates that we hit the txTimeout + // If we hit the txTimeout, replace the tx up to maxReplacements times + for i := tm.current.Replacements; err == context.DeadlineExceeded && i < tm.maxReplacements; i++ { + tx, err = tm.replace(tx) + // Do not attempt additional replacements if there was an error submitting this + // replacement tx + if err != nil { + break + } + receipt, err = tm.wait(tx) + } + + if receipt == nil { + txReceipt = types.Receipt{} + } else { + txReceipt = *(receipt) + } + var included includedTransaction + if errors.As(err, &included) { + txReceipt.TxHash = included.hash + } + + tm.deliver(&transactionReceipt{ + originTxHash: originHash, + Receipt: txReceipt, + err: err, + }) + + } +} + +func (tm *TransactionManager) deliver(result *transactionReceipt) { + if tm.Completed != nil { + tm.Completed(tm.Context(), result) + } +} + +// receiptAttempts lets upstream WaitMined observe any persisted predecessor. +// RPC validation still checks each returned receipt against its requested hash. +type receiptAttempts struct { + transactionSenderReader + hashes []ethcommon.Hash +} + +func (r receiptAttempts) TransactionReceipt(ctx context.Context, hash ethcommon.Hash) (*types.Receipt, error) { + if len(r.hashes) == 0 { + return r.transactionSenderReader.TransactionReceipt(ctx, hash) + } + var lookupErr error + for _, h := range r.hashes { + receipt, err := r.transactionSenderReader.TransactionReceipt(ctx, h) + if err == nil { + return receipt, nil + } + if !errors.Is(err, ethereum.NotFound) { + lookupErr = errors.Join(lookupErr, err) + } + } + if lookupErr != nil { + return nil, lookupErr + } + return nil, ethereum.NotFound +} + +func applyPriceBump(val *big.Int, priceBump uint64) *big.Int { + a := big.NewInt(100 + int64(priceBump)) + b := new(big.Int).Mul(a, val) + // div round up + b.Div(new(big.Int).Add(b, big.NewInt(99)), big.NewInt(100)) + // Geth also requires a strict increase, including for zero-valued tips. + if priceBump > 0 && b.Cmp(val) <= 0 { + b.Add(val, big.NewInt(1)) + } + return b +} + +func newReplacementTx(tx *types.Transaction) *types.Transaction { + var baseTx types.TxData + if tx.Type() == types.LegacyTxType { + // legacy tx, not London ready + baseTx = &types.LegacyTx{ + Nonce: tx.Nonce(), + GasPrice: applyPriceBump(tx.GasPrice(), priceBump), + Gas: tx.Gas(), + To: tx.To(), + Value: tx.Value(), + Data: tx.Data(), + } + } else { + baseTx = &types.DynamicFeeTx{ + ChainID: tx.ChainId(), AccessList: tx.AccessList(), + Nonce: tx.Nonce(), + // geth requires the price bump to be applied to both the gas tip cap and gas fee cap + GasFeeCap: applyPriceBump(tx.GasFeeCap(), priceBump), + GasTipCap: applyPriceBump(tx.GasTipCap(), priceBump), + Gas: tx.Gas(), + Value: tx.Value(), + Data: tx.Data(), + To: tx.To(), + } + } + + return types.NewTx(baseTx) +} + +func (r *transactionReceipt) OriginHash() ethcommon.Hash { return r.originTxHash } +func (r *transactionReceipt) Error() error { return r.err } + +type TransactionReceipt = transactionReceipt diff --git a/eth/transactionmanager_test.go b/eth/transactionmanager_test.go new file mode 100644 index 0000000..9f9f8b8 --- /dev/null +++ b/eth/transactionmanager_test.go @@ -0,0 +1,364 @@ +// Ported with the implementation from go-livepeer at 0ff08af1bb4cb268f428d619cc968b5eaee0d093. +// Fixtures and repetitive cases are consolidated; fee-threshold expectations +// and upstream assertion typos are adapted. Behavioral assertions are retained. +package eth + +import ( + "context" + "crypto/rand" + "errors" + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/livepeer/node/internal/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +type stubTransactionSenderReader struct { + err map[string]error + pending bool + tx *types.Transaction + receipt *types.Receipt + callsToTxByHash int + lookupErrors []error + minedAfter int +} + +func (stm *stubTransactionSenderReader) SendTransaction(ctx context.Context, tx *types.Transaction) error { + return stm.err["SendTransaction"] +} + +func (stm *stubTransactionSenderReader) TransactionByHash(ctx context.Context, txHash common.Hash) (tx *types.Transaction, isPending bool, err error) { + stm.callsToTxByHash++ + pending := stm.pending + if stm.minedAfter > 0 && stm.callsToTxByHash >= stm.minedAfter { + pending = false + } + if stm.callsToTxByHash <= len(stm.lookupErrors) { + return stm.tx, pending, stm.lookupErrors[stm.callsToTxByHash-1] + } + return stm.tx, pending, stm.err["TransactionByHash"] +} + +func (stm *stubTransactionSenderReader) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { + if stm.receipt == nil { + return nil, ethereum.NotFound + } + return stm.receipt, stm.err["TransactionReceipt"] +} + +func (stm *stubTransactionSenderReader) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) { + return []byte{}, stm.err["CodeAt"] +} + +type stubTransactionSigner struct { + err error +} + +func (sig *stubTransactionSigner) SignTx(tx *types.Transaction) (*types.Transaction, error) { + if sig.err != nil { + return nil, sig.err + } + return tx, nil +} + +func TestTransactionManager_SendTransaction(t *testing.T) { + backend := &stubTransactionSenderReader{err: map[string]error{"SendTransaction": errors.New("send failed")}} + tm := NewTransactionManager(backend, nil, nil, time.Second, 0) + defer tm.Stop() + tx := newStubLegacyTx(big.NewInt(10)) + require.ErrorIs(t, tm.SendTransaction(t.Context(), tx), backend.err["SendTransaction"]) + require.Empty(t, tm.queue, "a rejected send must not be tracked") + backend.err = nil + require.NoError(t, tm.SendTransaction(t.Context(), tx)) + require.Equal(t, []*types.Transaction{tx}, tm.queue) + next := newStubLegacyTx(big.NewInt(20)) + tm.Track(next) + require.Equal(t, []*types.Transaction{tx, next}, tm.queue, "Send and Track must enqueue in order") +} + +func TestTransactionManager_Wait(t *testing.T) { + assert := assert.New(t) + + eth := &stubTransactionSenderReader{ + err: make(map[string]error), + pending: true, + } + tm := NewTransactionManager(eth, nil, nil, 2*time.Second, 0) + defer tm.Stop() + + // Test error + // This calls bind.WaitMined() on the ethereum client, which will never actually return the actual underlying error and only log it using go-ethereum's custom logger + // The expected error should thus be 'context deadline exceeded' + // https://github.com/ethereum/go-ethereum/blob/aa637fd38a379db6da98df0d520fb1c5139a18ce/accounts/abi/bind/util.go#L41 + expErr := errors.New("context deadline exceeded") + + tx := types.NewTransaction(1, randomNonceAddress(), big.NewInt(100), 100000, big.NewInt(100), randomManagerBytes(68)) + + receipt, err := tm.wait(tx) + assert.Nil(receipt) + assert.EqualError(err, expErr.Error()) + + // No error, stub a receipt + eth.receipt = types.NewReceipt(randomManagerHash().Bytes(), false, 100000) + eth.err = nil + + receipt, err = tm.wait(tx) + assert.Equal(receipt.Status, uint64(1)) + assert.Equal(receipt.CumulativeGasUsed, uint64(100000)) + assert.Nil(err) +} + +func TestTransactionManager_Replace(t *testing.T) { + assert := assert.New(t) + + eth := &stubTransactionSenderReader{ + err: make(map[string]error), + } + baseFee := big.NewInt(9) + gasTipCap := big.NewInt(1) + gasFeeCap := new(big.Int).Add(gasTipCap, new(big.Int).Mul(baseFee, big.NewInt(2))) + gpm := &testFeeMonitor{ + maxGasPrice: big.NewInt(0), + } + tm := NewTransactionManager(eth, gpm, nil, 2*time.Second, 0) + defer tm.Stop() + + stubTx := newStubDynamicFeeTx(gasFeeCap, gasTipCap) + + // Test eth.TransactionByHash error + expErr := errors.New("TransactionByHash error") + eth.err["TransactionByHash"] = expErr + + tx, err := tm.replace(stubTx) + assert.Nil(tx) + assert.EqualError(err, expErr.Error()) + eth.err["TransactionByHash"] = nil + + // Test no error - ErrReplacingMinedTx + eth.pending = false + tx, err = tm.replace(stubTx) + assert.Nil(tx) + assert.EqualError(err, ErrReplacingMinedTx.Error()) + + // Test error is ethereum.NotFound - fail at next step + eth.pending = true + gpm.maxGasPrice = big.NewInt(1) + eth.err["TransactionByHash"] = ethereum.NotFound + tx, err = tm.replace(stubTx) + assert.Nil(tx) + assert.EqualError( + err, + fmt.Sprintf("replacement gas price exceeds max gas price suggested=%v max=%v", newReplacementTx(stubTx).GasFeeCap(), gpm.maxGasPrice), + ) + eth.err["TransactionByHash"] = nil + + // Replacement gas price exceeds max gas price + // Throw error + tx, err = tm.replace(stubTx) + assert.Nil(tx) + assert.EqualError( + err, + fmt.Sprintf("replacement gas price exceeds max gas price suggested=%v max=%v", newReplacementTx(stubTx).GasFeeCap(), gpm.maxGasPrice), + ) + + // Error signing replacement tx + expErr = errors.New("SignTx error") + sig := &stubTransactionSigner{ + err: nil, + } + tm.sig = sig + sig.err = expErr + gpm.maxGasPrice = big.NewInt(99999) + tx, err = tm.replace(stubTx) + assert.Nil(tx) + assert.EqualError(err, expErr.Error()) + + // Test when max gas price is nil - should still return signing replacement tx error + gpm.maxGasPrice = nil + tx, err = tm.replace(stubTx) + assert.Nil(tx) + assert.EqualError(err, expErr.Error()) + sig.err = nil + + // Error sending replacement tx + expErr = errors.New("SendTx error") + eth.err["SendTransaction"] = expErr + tx, err = tm.replace(stubTx) + assert.EqualError(err, expErr.Error()) + eth.err["SendTransaction"] = nil + + // Success + tx, err = tm.replace(stubTx) + assert.Nil(err) + expTx := newReplacementTx(stubTx) + assert.Equal(tx.Hash(), expTx.Hash()) +} + +func TestTransactionManager_CheckTxLoop(t *testing.T) { + lookupErr := errors.New("TransactionByHash error") + for _, tc := range []struct { + name string + mined bool + replacements, lookups int + minedAfter int + lookupErr, wantErr error + }{ + {name: "included", mined: true}, + {name: "timeout without replacements", lookups: 1, wantErr: context.DeadlineExceeded}, + {name: "replacement error", replacements: 1, lookups: 2, lookupErr: lookupErr, wantErr: lookupErr}, + {name: "replacement error stops attempts", replacements: 3, lookups: 2, lookupErr: lookupErr, wantErr: lookupErr}, + {name: "included before replacement", replacements: 1, lookups: 2, minedAfter: 2, wantErr: ErrReplacingMinedTx}, + {name: "replacement limit", replacements: 3, lookups: 10, wantErr: context.DeadlineExceeded}, + } { + t.Run(tc.name, func(t *testing.T) { + backend := &stubTransactionSenderReader{pending: true, err: map[string]error{"TransactionByHash": tc.lookupErr}} + // The first live check succeeds; the replacement itself can fail. + backend.lookupErrors = []error{nil} + backend.minedAfter = tc.minedAfter + if tc.mined { + backend.receipt = types.NewReceipt(randomManagerHash().Bytes(), false, 100000) + } + tm := NewTransactionManager(backend, &testFeeMonitor{maxGasPrice: big.NewInt(99999999)}, &stubTransactionSigner{}, time.Millisecond, tc.replacements) + sink := make(chan *transactionReceipt, 1) + tm.Completed = func(_ context.Context, result *transactionReceipt) { sink <- result } + done := make(chan struct{}) + go func() { defer close(done); tm.Start() }() + defer func() { + tm.Stop() + select { + case <-done: + case <-time.After(time.Second): + t.Error("transaction manager did not stop") + } + }() + tx := newStubLegacyTx(big.NewInt(10)) + require.NoError(t, tm.SendTransaction(t.Context(), tx)) + select { + case event := <-sink: + require.NotNil(t, event) + require.Equal(t, tx.Hash(), event.originTxHash) + require.ErrorIs(t, event.err, tc.wantErr) + require.Equal(t, tc.lookups, backend.callsToTxByHash) + if errors.Is(event.err, ErrReplacingMinedTx) { + require.Equal(t, tx.Hash(), event.TxHash) + } + if tc.mined { + require.Equal(t, backend.receipt, &event.Receipt) + } + case <-time.After(time.Second): + t.Fatal("transaction result was not delivered") + } + }) + } +} + +func TestApplyPriceBump(t *testing.T) { + for _, tc := range []struct{ value, bump, want int64 }{ + {500, 0, 500}, {500, 11, 555}, {500, 17, 585}, {500, 101, 1005}, + {50, 11, 56}, {501, 11, 557}, {9, 11, 10}, + } { + assert.Equal(t, big.NewInt(tc.want), applyPriceBump(big.NewInt(tc.value), uint64(tc.bump)), "value=%d bump=%d", tc.value, tc.bump) + } +} + +func TestNewReplacementTx(t *testing.T) { + for name, tx1 := range map[string]*types.Transaction{ + "legacy": newStubLegacyTx(big.NewInt(100)), + "dynamic": newStubDynamicFeeTx(big.NewInt(1000), big.NewInt(100)), + } { + t.Run(name, func(t *testing.T) { + tx2 := newReplacementTx(tx1) + assert.NotEqual(t, tx1.Hash(), tx2.Hash()) + assert.Equal(t, applyPriceBump(tx1.GasTipCap(), priceBump), tx2.GasTipCap()) + assert.Equal(t, applyPriceBump(tx1.GasFeeCap(), priceBump), tx2.GasFeeCap()) + assert.Equal(t, tx1.Nonce(), tx2.Nonce()) + assert.Equal(t, tx1.Gas(), tx2.Gas()) + assert.Equal(t, tx1.Value(), tx2.Value()) + assert.Equal(t, tx1.To(), tx2.To()) + assert.Equal(t, tx1.Data(), tx2.Data()) + }) + } +} + +func newStubLegacyTx(gasPrice *big.Int) *types.Transaction { + addr := randomNonceAddress() + return types.NewTx(&types.LegacyTx{ + Nonce: 1, + GasPrice: gasPrice, + Gas: 1000000, + Value: big.NewInt(100), + Data: randomManagerBytes(68), + To: &addr, + }) +} + +func newStubDynamicFeeTx(gasFeeCap, gasTipCap *big.Int) *types.Transaction { + addr := randomNonceAddress() + return types.NewTx(&types.DynamicFeeTx{ + Nonce: 1, + GasFeeCap: gasFeeCap, + GasTipCap: gasTipCap, + Gas: 1000000, + Value: big.NewInt(100), + Data: randomManagerBytes(68), + To: &addr, + }) +} + +// Upstream gas-monitor tests only need a mutable ceiling; production uses the +// existing Contracts fee ceiling instead of importing its gas monitor service. +type testFeeMonitor struct{ maxGasPrice *big.Int } + +func (m *testFeeMonitor) MaxGasPrice() *big.Int { return m.maxGasPrice } +func randomManagerBytes(n int) []byte { b := make([]byte, n); _, _ = rand.Read(b); return b } +func randomManagerHash() common.Hash { return common.BytesToHash(randomManagerBytes(32)) } + +func TestGethReplacementThresholdStrictlyIncreasesZero(t *testing.T) { + assert.Equal(t, big.NewInt(1), applyPriceBump(new(big.Int), priceBump)) + assert.Equal(t, big.NewInt(110), applyPriceBump(big.NewInt(100), priceBump)) + original := types.NewTx(&types.DynamicFeeTx{ChainID: big.NewInt(42161), Nonce: 7, GasFeeCap: big.NewInt(100), GasTipCap: new(big.Int), Gas: 21000, To: new(common.Address), AccessList: types.AccessList{{Address: common.HexToAddress("0x1234")}}}) + next := newReplacementTx(original) + assert.Equal(t, original.ChainId(), next.ChainId()) + assert.Equal(t, original.AccessList(), next.AccessList()) + assert.Equal(t, big.NewInt(1), next.GasTipCap()) +} + +func TestLegacyReplacementRetainsSignerChain(t *testing.T) { + path, password := test.WriteFixedKeystore(t) + key, err := OpenKeystoreFile(path, password) + require.NoError(t, err) + original, err := key.SignTxForChain(types.NewTx(&types.LegacyTx{Nonce: 7, GasPrice: big.NewInt(100), Gas: 21000, To: new(common.Address)}), big.NewInt(42161)) + require.NoError(t, err) + backend := &stubTransactionSenderReader{pending: true, err: map[string]error{}} + tm := NewTransactionManager(backend, &testFeeMonitor{}, key, time.Second, 1) + defer tm.Stop() + replacement, err := tm.replace(original) + require.NoError(t, err) + require.Equal(t, original.ChainId(), replacement.ChainId()) + from, err := types.Sender(types.LatestSignerForChainID(original.ChainId()), replacement) + require.NoError(t, err) + require.Equal(t, key.Address(), from) +} + +func TestStoppedManagerDoesNotDrainQueue(t *testing.T) { + tm := NewTransactionManager(&stubTransactionSenderReader{}, nil, nil, time.Hour, 0) + restores := 0 + tm.Restore = func(ctx context.Context, tx *types.Transaction) (TransactionRecovery, error) { + restores++ + return TransactionRecovery{}, ctx.Err() + } + for i := 0; i < 3; i++ { + tm.Track(types.NewTx(&types.LegacyTx{Nonce: uint64(i)})) + } + tm.Stop() + tm.Start() + require.Zero(t, restores, "stopped manager must not run restore callbacks for queued work") + require.Len(t, tm.queue, 3, "stopped manager must leave queued work intact") +} diff --git a/orchestrator/app.go b/orchestrator/app.go index b021b55..66767b2 100644 --- a/orchestrator/app.go +++ b/orchestrator/app.go @@ -33,36 +33,38 @@ import ( type Params struct { ConfigFile string `name:"config" configfile:"optional-default" basepath:"source" default:"config.toml" boa:"noconfig" descr:"Configuration file; empty disables discovery"` nodeconfig.Settings - Account *ethcommon.Address `descr:"Keystore account address"` - OrchestratorAddress *ethcommon.Address `descr:"Registered orchestrator address receiving ticket payments; defaults to the keystore account"` - ETHUSDFeed *ethcommon.Address `name:"eth-usd-feed" descr:"ETH/USD oracle address, alternative to a fixed wei-per-usd rate"` - PriceMaxAge time.Duration `default:"2h" descr:"Maximum age of the oracle observation"` - Listen netip.AddrPort `default:"127.0.0.1:8935" descr:"Public HTTP listener"` - MetricsListen netip.AddrPort `default:"127.0.0.1:8936" descr:"Loopback metrics listener"` - ServiceURL boa.Text[*url.URL] `optional:"true" descr:"Public orchestrator base URL; defaults to http://listen"` - RunnerServiceURL boa.Text[*url.URL] `optional:"true" descr:"Runner-facing base URL for callbacks and trickle; defaults to service-url"` - ProxyURLTemplate string `optional:"true" descr:"Generated proxy URL with {proxy} in a hostname label or final path segment"` - BootstrapSecret string `secret:"true" optional:"true" descr:"Dynamic runner bootstrap credential"` - BootstrapSecretFile string `basepath:"source" secretfor:"BootstrapSecret" descr:"File containing runner bootstrap credential"` - RunnerConfig string `basepath:"source" optional:"true" file:"true" descr:"Static runner TOML path"` - RedeemerDB string `optional:"true" file:"optional"` - KeystoreFile string `basepath:"source" optional:"true" descr:"Encrypted geth redemption account JSON file"` - KeystorePassword *string `secret:"true" optional:"true" descr:"Keystore decryption password"` - KeystorePasswordFile string `basepath:"source" secretfor:"KeystorePassword" descr:"Owner-only file containing the exact keystore password bytes"` - RPCURL *url.URL `name:"rpc-url" secret:"true" optional:"true"` - RPCURLFile string `basepath:"source" name:"rpc-url-file" secretfor:"RPCURL"` - ChainID *uint64 `min:"1"` - RedeemerMaxFeePerGas *uint256.Int `descr:"Optional maximum redemption fee in wei per gas"` - Controller *ethcommon.Address `name:"controller-address"` - WeiPerUSD *big.Rat - TicketFaceValue *uint256.Int - TicketWinProb *uint256.Int - RunnerGrants []string `optional:"true" descr:"Private runner IP[:port] or CIDR grants"` - SessionProxyGrants []string `optional:"true" descr:"Private session proxy IP[:port] or CIDR grants"` - HealthGrants []string `optional:"true" descr:"Private static runner health IP[:port] or CIDR grants"` - HeartbeatInterval time.Duration `default:"5s" descr:"Runner heartbeat interval"` - HeartbeatTTL time.Duration `default:"30s" descr:"Runner heartbeat expiry"` - PrintConfig bool `optional:"true" boa:"noconfig,noenv" descr:"Print audited redacted TOML configuration"` + Account *ethcommon.Address `descr:"Keystore account address"` + OrchestratorAddress *ethcommon.Address `descr:"Registered orchestrator address receiving ticket payments; defaults to the keystore account"` + ETHUSDFeed *ethcommon.Address `name:"eth-usd-feed" descr:"ETH/USD oracle address, alternative to a fixed wei-per-usd rate"` + PriceMaxAge time.Duration `default:"2h" descr:"Maximum age of the oracle observation"` + Listen netip.AddrPort `default:"127.0.0.1:8935" descr:"Public HTTP listener"` + MetricsListen netip.AddrPort `default:"127.0.0.1:8936" descr:"Loopback metrics listener"` + ServiceURL boa.Text[*url.URL] `optional:"true" descr:"Public orchestrator base URL; defaults to http://listen"` + RunnerServiceURL boa.Text[*url.URL] `optional:"true" descr:"Runner-facing base URL for callbacks and trickle; defaults to service-url"` + ProxyURLTemplate string `optional:"true" descr:"Generated proxy URL with {proxy} in a hostname label or final path segment"` + BootstrapSecret string `secret:"true" optional:"true" descr:"Dynamic runner bootstrap credential"` + BootstrapSecretFile string `basepath:"source" secretfor:"BootstrapSecret" descr:"File containing runner bootstrap credential"` + RunnerConfig string `basepath:"source" optional:"true" file:"true" descr:"Static runner TOML path"` + RedeemerDB string `optional:"true" file:"optional"` + KeystoreFile string `basepath:"source" optional:"true" descr:"Encrypted geth redemption account JSON file"` + KeystorePassword *string `secret:"true" optional:"true" descr:"Keystore decryption password"` + KeystorePasswordFile string `basepath:"source" secretfor:"KeystorePassword" descr:"Owner-only file containing the exact keystore password bytes"` + RPCURL *url.URL `name:"rpc-url" secret:"true" optional:"true"` + RPCURLFile string `basepath:"source" name:"rpc-url-file" secretfor:"RPCURL"` + ChainID *uint64 `min:"1"` + RedeemerTxTimeout time.Duration `default:"5m" descr:"Inclusion timeout for each redemption transaction attempt"` + RedeemerMaxTxReplacements int `default:"1" min:"0" descr:"Maximum fee replacements per redemption attempt"` + RedeemerMaxFeePerGas *uint256.Int `descr:"Optional maximum redemption fee in wei per gas"` + Controller *ethcommon.Address `name:"controller-address"` + WeiPerUSD *big.Rat + TicketFaceValue *uint256.Int + TicketWinProb *uint256.Int + RunnerGrants []string `optional:"true" descr:"Private runner IP[:port] or CIDR grants"` + SessionProxyGrants []string `optional:"true" descr:"Private session proxy IP[:port] or CIDR grants"` + HealthGrants []string `optional:"true" descr:"Private static runner health IP[:port] or CIDR grants"` + HeartbeatInterval time.Duration `default:"5s" descr:"Runner heartbeat interval"` + HeartbeatTTL time.Duration `default:"30s" descr:"Runner heartbeat expiry"` + PrintConfig bool `optional:"true" boa:"noconfig,noenv" descr:"Print audited redacted TOML configuration"` } func (p Params) paymentsRequested() bool { @@ -92,6 +94,9 @@ func (p Params) Validate() error { if p.TicketWinProb.Eq(new(uint256.Int).SetAllOne()) { return errors.New("--ticket-win-prob must be less than 2^256 - 1") } + if p.RedeemerTxTimeout <= 0 || p.RedeemerMaxTxReplacements < 0 { + return errors.New("--redeemer-tx-timeout must be positive and --redeemer-max-tx-replacements must be nonnegative") + } if p.RedeemerMaxFeePerGas != nil && p.RedeemerMaxFeePerGas.IsZero() { return errors.New("--redeemer-max-fee-per-gas must be positive") } @@ -193,6 +198,7 @@ func printConfig(ctx *boa.HookContext, p *Params, out io.Writer) error { "Listen": &p.Listen, "MetricsListen": &p.MetricsListen, "RunnerGrants": &p.RunnerGrants, "SessionProxyGrants": &p.SessionProxyGrants, "HealthGrants": &p.HealthGrants, "HeartbeatInterval": &p.HeartbeatInterval, "HeartbeatTTL": &p.HeartbeatTTL, + "RedeemerTxTimeout": &p.RedeemerTxTimeout, "RedeemerMaxTxReplacements": &p.RedeemerMaxTxReplacements, "RedeemerDB": &p.RedeemerDB, "ChainID": &p.ChainID, "RedeemerMaxFeePerGas": &p.RedeemerMaxFeePerGas, "Controller": &p.Controller, "WeiPerUSD": &p.WeiPerUSD, "ETHUSDFeed": &p.ETHUSDFeed, "PriceMaxAge": &p.PriceMaxAge, "TicketFaceValue": &p.TicketFaceValue, "TicketWinProb": &p.TicketWinProb, @@ -333,6 +339,11 @@ func Serve(parent context.Context, p Params, logOut io.Writer) (result error) { }) ctx, cancel := signal.NotifyContext(parent, syscall.SIGINT, syscall.SIGTERM) defer cancel() + if engine != nil { + if err := StartRedemptionRecovery(ctx, redeemerDB, paymentChain, redeemerKey, RedemptionOptions{Timeout: p.RedeemerTxTimeout, MaxReplacements: p.RedeemerMaxTxReplacements}); err != nil { + return err + } + } mainServer := &http.Server{BaseContext: func(net.Listener) context.Context { return ctx }, Handler: app, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 90 * time.Second, MaxHeaderBytes: 1 << 20, MaxHeaderValueCount: 128} metricsServer := &http.Server{Handler: metricsMux, ReadHeaderTimeout: 5 * time.Second, IdleTimeout: 30 * time.Second, MaxHeaderBytes: 1 << 16, MaxHeaderValueCount: 128} errs := make(chan error, 2) diff --git a/orchestrator/app_test.go b/orchestrator/app_test.go index 61f1031..4d41aec 100644 --- a/orchestrator/app_test.go +++ b/orchestrator/app_test.go @@ -365,6 +365,7 @@ func TestRedemptionValidationBeforeResources(t *testing.T) { func paymentKeystoreParams(t *testing.T, endpoint string) Params { t.Helper() return Params{ + RedeemerTxTimeout: 5 * time.Minute, RedeemerMaxTxReplacements: 1, Listen: netip.MustParseAddrPort("127.0.0.1:0"), MetricsListen: netip.MustParseAddrPort("127.0.0.1:0"), ServiceURL: boa.Text[*url.URL]{Value: mustURL(t, "http://127.0.0.1:8935")}, BootstrapSecret: "test", HeartbeatInterval: time.Hour, HeartbeatTTL: 2 * time.Hour, diff --git a/orchestrator/migrations/002_redemption_transactions.sql b/orchestrator/migrations/002_redemption_transactions.sql new file mode 100644 index 0000000..3ddd57c --- /dev/null +++ b/orchestrator/migrations/002_redemption_transactions.sql @@ -0,0 +1,41 @@ +-- UP +DROP INDEX redemption_nonce; +DROP INDEX winning_tickets_pending; +CREATE TABLE redemption_transactions ( + hash TEXT PRIMARY KEY, sig BLOB NOT NULL, + root_hash TEXT NOT NULL, replaces_hash TEXT, + raw_transaction BLOB NOT NULL, redeemer_address TEXT NOT NULL, nonce TEXT NOT NULL, + phase TEXT NOT NULL, error TEXT, prepared_at TEXT NOT NULL, broadcast_at TEXT, + acknowledged INTEGER NOT NULL DEFAULT 0 +); +CREATE INDEX redemption_transactions_ticket ON redemption_transactions(sig); +CREATE INDEX redemption_transactions_root ON redemption_transactions(root_hash); +CREATE INDEX redemption_transactions_nonce ON redemption_transactions(redeemer_address,nonce); +CREATE INDEX redemption_transactions_acknowledged ON redemption_transactions(redeemer_address,length(nonce) DESC,nonce DESC) WHERE acknowledged=1; +INSERT INTO redemption_transactions(hash,sig,root_hash,raw_transaction,redeemer_address,nonce,phase,error,prepared_at,broadcast_at,acknowledged) + SELECT w.tx_hash,a.sig,w.tx_hash,a.raw_transaction,a.redeemer_address,a.nonce,a.phase,a.error,a.attempted_at, + CASE WHEN a.phase!='prepared' THEN a.attempted_at END, + CASE WHEN a.phase IN ('submitted','confirmed','reverted') THEN 1 ELSE 0 END + FROM redemption_attempts a JOIN winning_tickets w ON w.sig=a.sig + WHERE w.tx_hash IS NOT NULL AND a.raw_transaction IS NOT NULL AND a.nonce IS NOT NULL; +-- DOWN +-- Refuse to discard identities that the old one-row-per-ticket schema cannot hold. +CREATE TEMP TABLE cannot_downgrade_redemption_transaction_history (preserved INTEGER); +CREATE TEMP TRIGGER protect_redemption_history BEFORE INSERT ON cannot_downgrade_redemption_transaction_history WHEN NEW.preserved=0 + BEGIN SELECT RAISE(ABORT,'cannot_downgrade_redemption_transaction_history: retain the upgraded database and use a compatible binary'); END; +INSERT INTO cannot_downgrade_redemption_transaction_history + SELECT CASE WHEN EXISTS(SELECT 1 FROM redemption_transactions GROUP BY sig HAVING count(*)>1) + OR EXISTS(SELECT 1 FROM redemption_attempts WHERE nonce IS NOT NULL GROUP BY redeemer_address,nonce HAVING count(*)>1) + OR EXISTS(SELECT 1 FROM redemption_attempts WHERE phase NOT IN ('prepared','broadcast','submitted','confirmed','reverted','expired')) + -- The old schema must reproduce the identity exactly on a later upgrade. + OR EXISTS(SELECT 1 FROM redemption_transactions t LEFT JOIN redemption_attempts a ON a.sig=t.sig LEFT JOIN winning_tickets w ON w.sig=t.sig + WHERE a.sig IS NULL OR t.hash IS NOT w.tx_hash OR t.root_hash!=t.hash OR t.replaces_hash IS NOT NULL + OR t.raw_transaction IS NOT a.raw_transaction OR t.redeemer_address IS NOT a.redeemer_address OR t.nonce IS NOT a.nonce + OR t.phase IS NOT a.phase OR t.error IS NOT a.error OR t.prepared_at IS NOT a.attempted_at + OR t.broadcast_at IS NOT (CASE WHEN a.phase!='prepared' THEN a.attempted_at END) + OR t.acknowledged!=(CASE WHEN a.phase IN ('submitted','confirmed','reverted') THEN 1 ELSE 0 END)) + THEN 0 ELSE 1 END; +DROP TABLE cannot_downgrade_redemption_transaction_history; +DROP TABLE redemption_transactions; +CREATE UNIQUE INDEX redemption_nonce ON redemption_attempts(redeemer_address,nonce) WHERE nonce IS NOT NULL; +CREATE INDEX winning_tickets_pending ON winning_tickets(payer_address,creation_round,seq) WHERE tx_hash IS NULL; diff --git a/orchestrator/redeemer.go b/orchestrator/redeemer.go index 614cd2b..c96db13 100644 --- a/orchestrator/redeemer.go +++ b/orchestrator/redeemer.go @@ -1,53 +1,43 @@ +// Ticket queue/retry policy ported from go-livepeer/pm/queue.go and +// pm/sendermonitor.go at 0ff08af1bb4cb268f428d619cc968b5eaee0d093, +// by Yondon Fu, Nico Vergauwen and Rafal Leszko. MIT notice in LICENSE. package orchestrator import ( "context" - "database/sql" "errors" "fmt" "math/big" "time" - ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common" "github.com/livepeer/node/eth" + "github.com/livepeer/node/pm" ) -// RedeemPending only reveals randomness once the parameter acceptance window -// has ended. Preparation failures leave tickets queued; signed identities are -// durable before broadcasting. Call from one bounded worker per store. -// The expiry rule follows Nico Vergauwen's go-livepeer/pm/queue.go; L1-clock -// handling follows Rafał Leszko's 4b6ede31f040a084ff9e55bd798a0d6fffee1e1b. -func RedeemPending(ctx context.Context, store *RedeemerDB, chain eth.PaymentChain, redeemer *eth.Key, chainID *big.Int, snapshot eth.ChainSnapshot) []error { - if store == nil || redeemer == nil || chainID == nil || snapshot.Block == nil || snapshot.Round == nil { - return []error{fmt.Errorf("redemption is not configured")} - } - // A crash after preparation but before the broadcast marker is safe to - // resume. A broadcast marker requires receipt checks or explicit retry. - var preparedHash string - err := store.db.QueryRow("SELECT w.tx_hash FROM winning_tickets w JOIN redemption_attempts a ON a.sig=w.sig WHERE a.phase='prepared' LIMIT 1").Scan(&preparedHash) - if err == nil { - if err := RetryRedemption(ctx, store, chain.Contracts, ethcommon.HexToHash(preparedHash)); err != nil { - return []error{err} - } - } else if !errors.Is(err, sql.ErrNoRows) { - return []error{err} - } - var blocked int - if err := store.db.QueryRowContext(ctx, "SELECT COUNT(*) FROM redemption_attempts WHERE phase IN ('prepared','broadcast')").Scan(&blocked); err != nil { - return []error{err} - } - if blocked != 0 { - return nil +// Used tickets and finalized failures settle through their chain observations. +// This is the terminal error that can arise during submission itself. +func isNonRetryableTicketErr(err error) bool { + return errors.Is(err, eth.ErrTicketCreationRound) +} + +// RedeemPending adapts handleBlockEvent and redeemWinningTicket to node snapshots. +// Inclusion/replacement waits run on the service manager, never on this heartbeat. +func RedeemPending(ctx context.Context, store *RedeemerDB, chain eth.PaymentChain, key *eth.Key, chainID *big.Int, snapshot eth.ChainSnapshot) []error { + if store == nil || key == nil || chainID == nil || chain.Contracts == nil || snapshot.Block == nil || snapshot.Round == nil { + return []error{errors.New("redemption is not configured")} } - floor, err := store.SavedNonceFloor(ctx, redeemer.Address()) + // The fallback manager is store-owned and joined by Close. Production starts + // it first under the service context with its configured recovery options. + recovery, err := store.recoveryFor(context.WithoutCancel(ctx), chain, key, defaultRedemptionOptions) if err != nil { return []error{err} } + failures := recovery.resume(ctx) payers, err := store.PendingPayers() if err != nil { - return []error{err} + return append(failures, err) } - var failures []error for _, payer := range payers { if ctx.Err() != nil { return append(failures, ctx.Err()) @@ -61,7 +51,7 @@ func RedeemPending(ctx context.Context, store *RedeemerDB, chain eth.PaymentChai continue } if ticket.CreationRound < snapshot.Round.Int64()-2 { - _, err := store.db.Exec("INSERT INTO redemption_attempts(sig,attempted_at,phase,error) VALUES(?,?,'expired','ticket creation round expired')", ticket.Sig, time.Now().UTC().Format(time.RFC3339Nano)) + err = store.setTicketPhase(ctx, ticket.Sig, "expired") if err != nil { failures = append(failures, err) } @@ -70,75 +60,113 @@ func RedeemPending(ctx context.Context, store *RedeemerDB, chain eth.PaymentChai if ticket.ParamsExpirationBlock.Cmp(snapshot.Block) > 0 { continue } - redeemTicket := eth.RedeemTicket{Recipient: ticket.Recipient, Sender: ticket.PayerAddress, FaceValue: ticket.FaceValue, WinProb: ticket.WinProb, SenderNonce: ticket.TicketNonce, RecipientRandHash: ticket.RecipientRandHash, AuxData: ticket.AuxData(), Signature: ticket.Sig, RecipientRand: ticket.RecipientRand} - if chain.Contracts == nil { - return append(failures, errors.New("redemption is not configured")) - } active, err := chain.IsActiveAt(ctx, ticket.Recipient, snapshot) if err != nil { - failures = append(failures, fmt.Errorf("payer %s: %w", payer.Hex(), err)) + failures = append(failures, err) continue } if !active { continue } - chain.Contracts.SetNonceFloor(redeemer.Address(), floor) - prepared, err := chain.PrepareRedemptionAndStore(ctx, redeemer, chainID, redeemTicket, func(prepared eth.SignedTransaction) error { - return store.recordPrepared(ctx, ticket, prepared) - }) - if err == nil { - err = broadcastRedemption(ctx, store, chain.Contracts, ticket.Sig, prepared) + used, err := chain.IsUsedTicket(ctx, ticket.Hash(), "latest") + if err != nil { + failures = append(failures, err) + continue + } + if used { + finalized, err := chain.IsUsedTicket(ctx, ticket.Hash(), "finalized") + if err == nil { + if finalized { + err = store.setTicketPhase(ctx, ticket.Sig, "used") + } else { + err = store.setTicketPhase(ctx, ticket.Sig, "awaiting_finality") + } + } + if err != nil { + failures = append(failures, err) + } + continue + } + err = chain.SubmitRedemption(ctx, recovery.nonces, recovery.manager, key, chainID, redeemTicket(ticket), func(tx eth.SignedTransaction) error { return store.recordPrepared(ctx, ticket, tx) }) + if errors.Is(err, errRecoveringNonce) { + continue } if err != nil { + if isNonRetryableTicketErr(err) { + err = errors.Join(err, store.setTicketPhase(ctx, ticket.Sig, "invalid")) + } failures = append(failures, fmt.Errorf("payer %s: %w", payer.Hex(), err)) } } return failures } +func redeemTicket(ticket *pm.SignedTicket) eth.RedeemTicket { + return eth.RedeemTicket{Recipient: ticket.Recipient, Sender: ticket.PayerAddress, FaceValue: ticket.FaceValue, WinProb: ticket.WinProb, SenderNonce: ticket.TicketNonce, RecipientRandHash: ticket.RecipientRandHash, AuxData: ticket.AuxData(), Signature: ticket.Sig, RecipientRand: ticket.RecipientRand} +} +func (s *RedeemerDB) setTicketPhase(ctx context.Context, sig []byte, phase string) error { + if terminalRedemption(phase) { + return s.finishTicket(ctx, sig, phase, "") + } + _, err := s.db.ExecContext(ctx, `INSERT INTO redemption_attempts(sig,attempted_at,phase) VALUES(?,?,?) ON CONFLICT(sig) DO UPDATE SET phase=excluded.phase WHERE redemption_attempts.phase IN ('retryable','awaiting_finality')`, sig, time.Now().UTC().Format(time.RFC3339Nano), phase) + return err +} type ReceiptReader interface { - Receipt(context.Context, ethcommon.Hash) (confirmed, reverted bool, err error) + Receipt(context.Context, common.Hash) (confirmed, reverted bool, err error) } -// ReconcileSubmitted checks receipts once per background tick. It never -// broadcasts, so a pending or uncertain transaction cannot be sent twice. +// ReconcileSubmitted checks every retained identity, including old retry groups. +// A finalized success wins over sibling failures; failed attempts alone cannot +// release liability while another signed identity might still redeem the ticket. func ReconcileSubmitted(ctx context.Context, store *RedeemerDB, chain ReceiptReader) []error { items, err := store.SubmittedRedemptions() if err != nil { return []error{err} } var failures []error + successes := map[string]bool{} + reverts := map[string]common.Hash{} + pending := map[common.Hash]bool{} for _, item := range items { confirmed, reverted, err := chain.Receipt(ctx, item.Hash) + id := string(item.Signature) if err != nil { failures = append(failures, fmt.Errorf("receipt %s: %w", item.Hash.Hex(), err)) continue } if confirmed { - err = store.ConfirmRedemption(item.Signature) - } - if reverted { - err = store.FailRedemption(item.Signature) - failures = append(failures, fmt.Errorf("transaction %s reverted", item.Hash.Hex())) + successes[id] = true + err = store.finishTicket(ctx, item.Signature, "confirmed", item.Hash.Hex()) + } else if reverted { + reverts[id] = item.Hash + err = store.finishTicket(ctx, item.Signature, "reverted", item.Hash.Hex()) + } else { + pending[item.Hash] = true } if err != nil { failures = append(failures, err) } } + for id, hash := range reverts { + if successes[id] { + continue + } + failures = append(failures, fmt.Errorf("transaction failed: redemption %s reverted", hash.Hex())) + } + if reader, ok := chain.(finalizedNonceReader); ok { + failures = append(failures, store.reconcileFinalizedNonces(ctx, reader, pending)...) + } return failures } - -func ProcessRedemptions(ctx context.Context, store *RedeemerDB, chain eth.PaymentChain, redeemer *eth.Key, chainID *big.Int) []error { - var failures []error - if store == nil || redeemer == nil { +func ProcessRedemptions(ctx context.Context, store *RedeemerDB, chain eth.PaymentChain, key *eth.Key, chainID *big.Int) []error { + if store == nil || key == nil { return nil } + // Reconcile before admission so a successful predecessor prevents resubmission. + failures := ReconcileSubmitted(ctx, store, chain) snapshot, err := chain.Snapshot(ctx) if err != nil { - failures = append(failures, err) - } else { - failures = append(failures, RedeemPending(ctx, store, chain, redeemer, chainID, snapshot)...) + return append(failures, err) } - failures = append(failures, ReconcileSubmitted(ctx, store, chain)...) - return failures + return append(failures, RedeemPending(ctx, store, chain, key, chainID, snapshot)...) } diff --git a/orchestrator/redeemer_db.go b/orchestrator/redeemer_db.go index 61ebd51..1c37680 100644 --- a/orchestrator/redeemer_db.go +++ b/orchestrator/redeemer_db.go @@ -15,7 +15,7 @@ import ( "os" "path/filepath" "strings" - "time" + "sync" ethcommon "github.com/ethereum/go-ethereum/common" "github.com/livepeer/node/nodeconfig/migrations" @@ -28,7 +28,11 @@ var redeemerMigrationFiles embed.FS // RedeemerDB owns the orchestrator redeemer's durable winning-ticket queue, // redemption lifecycle, and chain activity observations. -type RedeemerDB struct{ db *sql.DB } +type RedeemerDB struct { + db *sql.DB + recoveryMu sync.Mutex + recovery *redemptionRecovery +} func OpenRedeemerDB(path string) (*RedeemerDB, error) { ctx := context.Background() @@ -94,7 +98,15 @@ func openRedeemerDB(ctx context.Context, path string, create bool) (*sql.DB, err return db, nil } -func (s *RedeemerDB) Close() error { return s.db.Close() } +func (s *RedeemerDB) Close() error { + s.recoveryMu.Lock() + r := s.recovery + s.recoveryMu.Unlock() + if r != nil { + r.close() + } + return s.db.Close() +} func (s *RedeemerDB) StoreWinningTicket(t *pm.SignedTicket) error { if err := validStoredTicket(t); err != nil { @@ -120,7 +132,7 @@ func (s *RedeemerDB) storeWinningTickets(ctx context.Context, payer ethcommon.Ad // Once redemption can reveal randomness, this epoch must never accept // more tickets, even if a reorg moves the observed L1 clock backwards. var exposed int - if err := tx.QueryRow(`SELECT EXISTS(SELECT 1 FROM winning_tickets w JOIN redemption_attempts a ON a.sig=w.sig WHERE w.recipient_rand_hash=? AND a.phase!='expired')`, randHash.Hex()).Scan(&exposed); err != nil { + if err := tx.QueryRow(`SELECT EXISTS(SELECT 1 FROM winning_tickets w JOIN redemption_attempts a ON a.sig=w.sig WHERE w.recipient_rand_hash=? AND (a.phase!='expired' OR EXISTS(SELECT 1 FROM redemption_transactions t WHERE t.sig=w.sig)))`, randHash.Hex()).Scan(&exposed); err != nil { return err } if exposed != 0 { @@ -135,7 +147,9 @@ func (s *RedeemerDB) storeWinningTickets(ctx context.Context, payer ethcommon.Ad } // Include every unconfirmed winner, across all sessions for this payer. // Keep uncertain transactions reserved until finalized receipt reconciliation. - rows, err := tx.Query("SELECT face_value FROM winning_tickets WHERE payer_address=? AND redeemed_at IS NULL AND creation_round>=? AND sig NOT IN (SELECT sig FROM redemption_attempts WHERE phase='reverted')", payer.Hex(), minRound) + rows, err := tx.Query(`SELECT face_value FROM winning_tickets WHERE payer_address=? AND redeemed_at IS NULL AND ( + (creation_round>=? AND sig NOT IN (SELECT sig FROM redemption_attempts WHERE phase IN ('used','invalid','expired','reverted'))) + OR EXISTS(SELECT 1 FROM redemption_transactions t WHERE t.sig=winning_tickets.sig AND t.phase NOT IN ('confirmed','reverted','superseded')))`, payer.Hex(), minRound) if err != nil { return err } @@ -170,32 +184,42 @@ func validStoredTicket(t *pm.SignedTicket) error { } func (s *RedeemerDB) SelectEarliestWinningTicket(payer ethcommon.Address, minCreationRound int64) (*pm.SignedTicket, error) { - row := s.db.QueryRow(`SELECT recipient,face_value,win_prob,ticket_nonce,recipient_rand,recipient_rand_hash,sig,creation_round,creation_round_block_hash,params_expiration_block - FROM winning_tickets WHERE payer_address=? AND creation_round>=? AND tx_hash IS NULL AND sig NOT IN (SELECT sig FROM redemption_attempts) ORDER BY seq LIMIT 1`, payer.Hex(), minCreationRound) - var recipient, randHash, blockHash, expiration string + row := s.db.QueryRow(`SELECT payer_address,recipient,face_value,win_prob,ticket_nonce,recipient_rand,recipient_rand_hash,sig,creation_round,creation_round_block_hash,params_expiration_block + FROM winning_tickets WHERE payer_address=? AND creation_round>=? AND redeemed_at IS NULL AND sig NOT IN (SELECT sig FROM redemption_attempts WHERE phase!='retryable') AND NOT EXISTS(SELECT 1 FROM winning_tickets other JOIN redemption_attempts a ON a.sig=other.sig WHERE other.payer_address=winning_tickets.payer_address AND a.phase IN ('prepared','broadcast','submitted','mined','awaiting_finality')) ORDER BY seq LIMIT 1`, payer.Hex(), minCreationRound) + ticket, err := scanStoredTicket(row) + if errors.Is(err, sql.ErrNoRows) { + return nil, nil + } + return ticket, err +} + +func scanStoredTicket(row *sql.Row) (*pm.SignedTicket, error) { + var payer, recipient, randHash, blockHash, expiration string var face, prob, rand, sig []byte var nonce int64 var round int64 - if err := row.Scan(&recipient, &face, &prob, &nonce, &rand, &randHash, &sig, &round, &blockHash, &expiration); err != nil { - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } + if err := row.Scan(&payer, &recipient, &face, &prob, &nonce, &rand, &randHash, &sig, &round, &blockHash, &expiration); err != nil { return nil, err } exp, ok := new(big.Int).SetString(expiration, 10) if !ok || nonce < 0 || nonce > 1<<32-1 { return nil, errors.New("corrupt winning ticket") } - return &pm.SignedTicket{Ticket: &pm.Ticket{PayerAddress: payer, Recipient: ethcommon.HexToAddress(recipient), + return &pm.SignedTicket{Ticket: &pm.Ticket{PayerAddress: ethcommon.HexToAddress(payer), Recipient: ethcommon.HexToAddress(recipient), FaceValue: new(big.Int).SetBytes(face), WinProb: new(big.Int).SetBytes(prob), TicketNonce: uint32(nonce), RecipientRandHash: ethcommon.HexToHash(randHash), CreationRound: round, CreationRoundBlockHash: ethcommon.HexToHash(blockHash), ParamsExpirationBlock: exp}, Sig: sig, RecipientRand: new(big.Int).SetBytes(rand)}, nil } +func (s *RedeemerDB) winningTicket(ctx context.Context, sig []byte) (*pm.SignedTicket, error) { + row := s.db.QueryRowContext(ctx, `SELECT payer_address,recipient,face_value,win_prob,ticket_nonce,recipient_rand,recipient_rand_hash,sig,creation_round,creation_round_block_hash,params_expiration_block FROM winning_tickets WHERE sig=?`, sig) + return scanStoredTicket(row) +} + func (s *RedeemerDB) WinningTicketCount(payer ethcommon.Address, minCreationRound int64) (int, error) { var count int - err := s.db.QueryRow(`SELECT COUNT(*) FROM winning_tickets WHERE payer_address=? AND creation_round>=? AND tx_hash IS NULL`, payer.Hex(), minCreationRound).Scan(&count) + err := s.db.QueryRow(`SELECT COUNT(*) FROM winning_tickets WHERE payer_address=? AND creation_round>=? AND redeemed_at IS NULL AND sig NOT IN (SELECT sig FROM redemption_attempts WHERE phase!='retryable')`, payer.Hex(), minCreationRound).Scan(&count) return count, err } @@ -205,65 +229,26 @@ type SubmittedRedemption struct { } func (s *RedeemerDB) SubmittedRedemptions() ([]SubmittedRedemption, error) { - rows, err := s.db.Query(`SELECT w.sig,w.tx_hash FROM winning_tickets w JOIN redemption_attempts a ON a.sig=w.sig WHERE w.tx_hash IS NOT NULL AND w.redeemed_at IS NULL AND a.phase NOT IN ('confirmed','reverted','expired')`) + rows, err := s.db.Query(`SELECT t.sig,t.hash FROM redemption_transactions t JOIN redemption_attempts a ON a.sig=t.sig WHERE a.phase NOT IN ('confirmed','used') AND t.phase NOT IN ('confirmed','reverted','superseded') ORDER BY t.rowid`) if err != nil { return nil, err } defer rows.Close() - var result []SubmittedRedemption + var out []SubmittedRedemption for rows.Next() { - var sig []byte - var raw string - if err := rows.Scan(&sig, &raw); err != nil { + var item SubmittedRedemption + var hash string + if err := rows.Scan(&item.Signature, &hash); err != nil { return nil, err } - if !ethcommon.IsHexHash(raw) { + if !ethcommon.IsHexHash(hash) { return nil, errors.New("corrupt redemption transaction hash") } - result = append(result, SubmittedRedemption{Signature: sig, Hash: ethcommon.HexToHash(raw)}) - } - return result, rows.Err() -} - -func (s *RedeemerDB) ConfirmRedemption(sig []byte) error { - tx, err := s.db.Begin() - if err != nil { - return err - } - defer tx.Rollback() - result, err := tx.Exec(`UPDATE winning_tickets SET redeemed_at=? WHERE sig=? AND tx_hash IS NOT NULL AND redeemed_at IS NULL`, time.Now().UTC().Format(time.RFC3339Nano), sig) - if err != nil { - return err + item.Hash = ethcommon.HexToHash(hash) + out = append(out, item) } - n, err := result.RowsAffected() - if err != nil { - return err - } - if n != 1 { - return errors.New("redemption is absent or already confirmed") - } - _, err = tx.Exec("UPDATE redemption_attempts SET phase='confirmed',error=NULL WHERE sig=?", sig) - if err != nil { - return err - } - return tx.Commit() + return out, rows.Err() } - -func (s *RedeemerDB) FailRedemption(sig []byte) error { - result, err := s.db.Exec(`UPDATE redemption_attempts SET phase='reverted',error='transaction reverted' WHERE sig=? AND phase!='reverted'`, sig) - if err != nil { - return err - } - n, err := result.RowsAffected() - if err != nil { - return err - } - if n != 1 { - return errors.New("redemption attempt is absent or already failed") - } - return nil -} - func (s *RedeemerDB) RemoveWinningTicket(t *pm.SignedTicket) error { if t == nil || t.Ticket == nil || len(t.Sig) == 0 { return errors.New("incomplete signed ticket") @@ -273,7 +258,7 @@ func (s *RedeemerDB) RemoveWinningTicket(t *pm.SignedTicket) error { } func (s *RedeemerDB) PendingPayers() ([]ethcommon.Address, error) { - rows, err := s.db.Query(`SELECT DISTINCT payer_address FROM winning_tickets WHERE tx_hash IS NULL AND sig NOT IN (SELECT sig FROM redemption_attempts)`) + rows, err := s.db.Query(`SELECT DISTINCT payer_address FROM winning_tickets WHERE redeemed_at IS NULL AND sig NOT IN (SELECT sig FROM redemption_attempts WHERE phase!='retryable')`) if err != nil { return nil, err } diff --git a/orchestrator/redeemer_db_test.go b/orchestrator/redeemer_db_test.go index 1a53273..c1372e9 100644 --- a/orchestrator/redeemer_db_test.go +++ b/orchestrator/redeemer_db_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "testing" + "time" ethcommon "github.com/ethereum/go-ethereum/common" "github.com/livepeer/node/eth" @@ -113,3 +114,56 @@ func TestRedemptionReceiptSettlesAttempt(t *testing.T) { }) } } + +func TestRedemptionMigrationHistoryAndRollback(t *testing.T) { + _, s, ticket, c, key := newRecoveryTest(t, 9) + files, err := fs.Sub(redeemerMigrationFiles, "migrations") + require.NoError(t, err) + require.NoError(t, migrations.Down(t.Context(), s.db, files)) + // Seed a genuine v1 database, then upgrade without changing its ticket identity. + prepared, err := c.PrepareRedemption(t.Context(), key, big.NewInt(1), redeemTicket(ticket)) + require.NoError(t, err) + _, err = s.db.Exec(`INSERT INTO redemption_attempts(sig,attempted_at,phase,raw_transaction,redeemer_address,nonce) VALUES(?,?,'submitted',?,?,?)`, ticket.Sig, time.Now().UTC().Format(time.RFC3339Nano), prepared.Raw, prepared.From.Hex(), "0") + require.NoError(t, err) + _, err = s.db.Exec("UPDATE winning_tickets SET tx_hash=? WHERE sig=?", prepared.Hash.Hex(), ticket.Sig) + require.NoError(t, err) + require.NoError(t, migrations.Up(t.Context(), s.db, files)) + history, err := s.Redemptions() + require.NoError(t, err) + require.Equal(t, []RedemptionStatus{{Hash: prepared.Hash.Hex(), Phase: "submitted"}}, history) + // A lossless rollback remains possible with the original representable state. + require.NoError(t, migrations.Down(t.Context(), s.db, files)) + require.NoError(t, migrations.Up(t.Context(), s.db, files)) + _, err = s.db.Exec("UPDATE redemption_attempts SET phase='retryable' WHERE sig=?", ticket.Sig) + require.NoError(t, err) + next := prepared + next.Hash = ethcommon.HexToHash("0xabcdef") + next.Nonce = 1 + require.NoError(t, s.recordPrepared(t.Context(), ticket, next)) + require.ErrorContains(t, migrations.Down(t.Context(), s.db, files), "cannot_downgrade_redemption_transaction_history") + history, err = s.Redemptions() + require.NoError(t, err) + require.Len(t, history, 2) +} + +func TestRedemptionRollbackPreservesTransactionProgress(t *testing.T) { + for _, phase := range []string{"submitted", "expired"} { + t.Run(phase, func(t *testing.T) { + _, s, ticket, c, key := newRecoveryTest(t, 22) + tickRecovery(t, s, c, key) + s = reopenRecoveryStore(t, s) + if phase == "expired" { + require.NoError(t, s.setTicketPhase(t.Context(), ticket.Sig, phase)) + } + floor, err := s.SavedNonceFloor(t.Context(), key.Address()) + require.NoError(t, err) + require.EqualValues(t, 1, floor) + files, err := fs.Sub(redeemerMigrationFiles, "migrations") + require.NoError(t, err) + require.ErrorContains(t, migrations.Down(t.Context(), s.db, files), "cannot_downgrade_redemption_transaction_history") + floor, err = s.SavedNonceFloor(t.Context(), key.Address()) + require.NoError(t, err) + require.EqualValues(t, 1, floor, "protected rollback must leave history intact") + }) + } +} diff --git a/orchestrator/redeemer_test.go b/orchestrator/redeemer_test.go new file mode 100644 index 0000000..ef55105 --- /dev/null +++ b/orchestrator/redeemer_test.go @@ -0,0 +1,77 @@ +package orchestrator + +import ( + "context" + "fmt" + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/livepeer/node/eth" + "github.com/stretchr/testify/require" +) + +func TestRedemptionEligibility(t *testing.T) { + for _, tc := range []struct { + block, round int64 + active, lookupError bool + }{{9, 5, true, false}, {10, 5, true, false}, {11, 5, true, false}, {10, 5, false, false}, {10, 5, true, true}, {10, 8, true, false}} { + t.Run(fmt.Sprintf("block=%d/round=%d/active=%t/error=%t", tc.block, tc.round, tc.active, tc.lookupError), func(t *testing.T) { + f, store, ticket, chain, key := newRecoveryTest(t, 1) + f.block, f.round, f.inactive, f.activeError = tc.block, tc.round, !tc.active, tc.lookupError + _, err := store.db.Exec("UPDATE winning_tickets SET creation_round=5") + require.NoError(t, err) + f.onSend = func(tx *types.Transaction) { + items, err := store.Redemptions() + require.NoError(t, err) + require.Equal(t, []RedemptionStatus{{Hash: tx.Hash().Hex(), Phase: "broadcast"}}, items, "persist the signed identity before sending") + } + failures := ProcessRedemptions(t.Context(), store, chain, key, big.NewInt(1)) + require.Equal(t, tc.lookupError, len(failures) != 0, "%v", failures) + expired := tc.round > 7 + redeemed := tc.block >= 10 && !expired && tc.active && !tc.lookupError + require.Equal(t, redeemed, len(f.transactions()) == 1) + if tc.block >= 10 && !expired { + f.mu.Lock() + recipient := f.activeRecipient + f.mu.Unlock() + require.Equal(t, ticket.Recipient, recipient, "activity lookup must use the ticket recipient") + } + queued, err := store.PendingPayers() + require.NoError(t, err) + require.Equal(t, !redeemed && !expired, len(queued) == 1) + }) + } +} + +func TestTerminalRedemptionSubmissionError(t *testing.T) { + f, store, ticket, chain, key := newRecoveryTest(t, 2) + f.prepareError = "private endpoint and calldata: " + eth.ErrTicketCreationRound.Error() + failures := ProcessRedemptions(t.Context(), store, chain, key, big.NewInt(1)) + require.Len(t, failures, 1) + require.ErrorIs(t, failures[0], eth.ErrTicketCreationRound) + require.NotContains(t, failures[0].Error(), "private") + require.Equal(t, "invalid", ticketPhase(t, store, ticket.Sig)) + queued, err := store.PendingPayers() + require.NoError(t, err) + require.Empty(t, queued) + require.Empty(t, f.transactions()) + store = reopenRecoveryStore(t, store) + f.mu.Lock() + f.prepareError = "" + f.mu.Unlock() + tickRecovery(t, store, chain, key) + require.Equal(t, "invalid", ticketPhase(t, store, ticket.Sig)) + require.Empty(t, f.transactions(), "a terminal preparation failure must not be submitted after restart") +} + +// Upstream queue/sendermonitor tests: terminal classification and retryable +// submission/CheckTx failures retain their assertions against the node adapter. +func TestUpstreamTicketRetryClassification(t *testing.T) { + for _, tc := range []struct { + err error + terminal bool + }{{nil, false}, {eth.ErrTicketCreationRound, true}, {fmt.Errorf("wrapped: %w", eth.ErrTicketCreationRound), true}, {fmt.Errorf("transaction failed txHash=abc"), false}, {fmt.Errorf("nonce too low"), false}, {fmt.Errorf("replacement transaction underpriced"), false}, {context.DeadlineExceeded, false}, {fmt.Errorf("checktx error"), false}} { + require.Equal(t, tc.terminal, isNonRetryableTicketErr(tc.err)) + } +} diff --git a/orchestrator/redemption_nonce.go b/orchestrator/redemption_nonce.go new file mode 100644 index 0000000..6ea7f54 --- /dev/null +++ b/orchestrator/redemption_nonce.go @@ -0,0 +1,176 @@ +package orchestrator + +import ( + "context" + "errors" + "fmt" + "strconv" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" +) + +var errRecoveringNonce = errors.New("recovering an outstanding redemption nonce") + +type finalizedNonceReader interface { + FinalizedNonce(context.Context, common.Address) (uint64, error) + InclusionReceipt(context.Context, common.Hash) (*types.Receipt, error) +} + +// repairNonceGap runs under the account lock. Acknowledged reservations remain +// monotonic: repair missing slots with their saved bytes instead of lowering the +// cursor on a possibly stale RPC observation. Ticket settlement at another nonce +// does not prove that a superseded identity consumed this slot. +func (r *redemptionRecovery) repairNonceGap(ctx context.Context, address common.Address, pending, floor uint64) (uint64, error) { + // Finalized state may be ahead of every local identity, even when all saved + // slots already have receipts. Carry that observation into fresh allocation. + finalized, err := r.chain.FinalizedNonce(ctx, address) + if err != nil { + return floor, err + } + allocated := max(pending, floor, finalized) + pending = max(pending, finalized) + for ; pending < floor; pending++ { + rows, err := r.store.db.QueryContext(ctx, `SELECT hash,root_hash,sig,raw_transaction,phase FROM redemption_transactions WHERE redeemer_address=? AND nonce=? ORDER BY rowid DESC`, address.Hex(), strconv.FormatUint(pending, 10)) + if err != nil { + return allocated, err + } + type saved struct { + hash, root, phase string + sig, raw []byte + } + var identities []saved + for rows.Next() { + var item saved + if err = rows.Scan(&item.hash, &item.root, &item.sig, &item.raw, &item.phase); err != nil { + break + } + identities = append(identities, item) + } + err = errors.Join(err, rows.Err(), rows.Close()) + if err != nil { + return allocated, err + } + present := false + for _, item := range identities { + if item.phase == "confirmed" || item.phase == "reverted" { + present = true // A canonical finalized receipt consumed this nonce. + break + } + hash := common.HexToHash(item.hash) + if _, err = r.chain.InclusionReceipt(ctx, hash); err == nil { + present = true + break + } else if !errors.Is(err, ethereum.NotFound) { + return allocated, err + } + if _, _, err = r.TransactionByHash(ctx, hash); err == nil { + present = true + break + } else if !errors.Is(err, ethereum.NotFound) { + return allocated, err + } + } + if present { + continue + } + if len(identities) == 0 { + return allocated, fmt.Errorf("missing saved redemption at nonce %d; restore its signed transaction before submitting more", pending) + } + item := identities[0] + r.mu.Lock() + active := r.tracking[common.HexToHash(item.root)] + r.mu.Unlock() + if active { + return allocated, errRecoveringNonce // The existing timeout/replacement lifecycle owns it. + } + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(item.raw); err != nil { + return allocated, errors.New("invalid stored redemption transaction") + } + identity, err := signedIdentity(tx) + if err != nil || identity.Hash.Hex() != item.hash || identity.From != address || identity.Nonce != pending { + return allocated, errors.New("stored redemption identity does not match its signed bytes") + } + if err := broadcastSavedRedemption(ctx, r.store, r.chain.Contracts, item.sig, identity, true); err != nil { + return allocated, err + } + return allocated, errRecoveringNonce + } + return allocated, nil +} + +// A finalized account count can resolve a missing local hash consumed by another +// process. It cannot establish ticket success, or resolve an unfinalized receipt. +func (s *RedeemerDB) reconcileFinalizedNonces(ctx context.Context, chain finalizedNonceReader, pending map[common.Hash]bool) []error { + if len(pending) == 0 { + return nil + } + rows, err := s.db.QueryContext(ctx, `SELECT t.hash,t.redeemer_address,t.nonce FROM redemption_transactions t JOIN redemption_attempts a ON a.sig=t.sig WHERE a.phase NOT IN ('confirmed','used') AND t.phase NOT IN ('confirmed','reverted','superseded')`) + if err != nil { + return []error{err} + } + type unresolved struct { + hash common.Hash + nonce uint64 + } + bySender := map[common.Address][]unresolved{} + for rows.Next() { + var hash, sender, nonce string + if err = rows.Scan(&hash, &sender, &nonce); err != nil { + break + } + if !pending[common.HexToHash(hash)] { + continue + } + var n uint64 + n, err = strconv.ParseUint(nonce, 10, 64) + if err != nil || !common.IsHexAddress(sender) { + err = errors.New("invalid stored redemption account or nonce") + break + } + address := common.HexToAddress(sender) + bySender[address] = append(bySender[address], unresolved{common.HexToHash(hash), n}) + } + err = errors.Join(err, rows.Err(), rows.Close()) + if err != nil { + return []error{err} + } + var failures []error + for address, items := range bySender { + count, err := chain.FinalizedNonce(ctx, address) + if err != nil { + failures = append(failures, err) + continue + } + for _, item := range items { + if item.nonce >= count { + continue + } + // Receipt's boolean result includes unfinalized inclusion. Only an + // actual missing/orphaned receipt permits external-nonce retirement. + if _, err := chain.InclusionReceipt(ctx, item.hash); !errors.Is(err, ethereum.NotFound) { + if err != nil { + failures = append(failures, err) + } + continue + } + tx, err := s.db.BeginTx(ctx, nil) + if err == nil { + _, err = tx.ExecContext(ctx, `UPDATE redemption_transactions SET phase='superseded',error=NULL WHERE hash=? AND phase NOT IN ('confirmed','reverted','superseded')`, item.hash.Hex()) + if err == nil { + _, err = tx.ExecContext(ctx, `UPDATE redemption_attempts SET phase='retryable',error=NULL WHERE phase IN ('prepared','broadcast','submitted','mined') AND sig IN (SELECT sig FROM winning_tickets WHERE tx_hash=?)`, item.hash.Hex()) + } + if err == nil { + err = tx.Commit() + } + tx.Rollback() + } + if err != nil { + failures = append(failures, err) + } + } + } + return failures +} diff --git a/orchestrator/redemption_nonce_test.go b/orchestrator/redemption_nonce_test.go new file mode 100644 index 0000000..3f17079 --- /dev/null +++ b/orchestrator/redemption_nonce_test.go @@ -0,0 +1,246 @@ +package orchestrator + +import ( + "fmt" + "math/big" + "testing" + "time" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/livepeer/node/pm" + "github.com/stretchr/testify/require" +) + +func TestDroppedRedemptionRefillsNonceAfterRestart(t *testing.T) { + for _, phase := range []string{"retryable", "expired", "invalid", "used"} { + t.Run(phase, func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 230) + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, RedemptionOptions{Timeout: 30 * time.Millisecond, MaxReplacements: 0})) + tickRecovery(t, s, c, key) + original := f.transactions()[0] + f.mu.Lock() + delete(f.pool, 0) + f.mu.Unlock() + require.Eventually(t, func() bool { return ticketPhase(t, s, ticket.Sig) == "retryable" }, time.Second, time.Millisecond) + s = reopenRecoveryStore(t, s) + if phase != "retryable" { + require.NoError(t, s.setTicketPhase(t.Context(), ticket.Sig, phase)) + } + // A failed claim must prevent both the saved-byte replay and any fresh + // higher nonce. Restart retains the original timestamp and identity. + _, err := s.db.Exec(`CREATE TRIGGER reject_gap_send BEFORE UPDATE OF phase ON redemption_transactions BEGIN SELECT RAISE(ABORT,'injected persistence failure'); END`) + require.NoError(t, err) + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + require.Len(t, f.transactions(), 1) + _, err = s.db.Exec("DROP TRIGGER reject_gap_send") + require.NoError(t, err) + tickRecovery(t, s, c, key) + sent := f.transactions() + require.Len(t, sent, 2) + require.Equal(t, original.Hash(), sent[1].Hash(), "refill the evicted slot with the exact saved bytes") + if phase != "retryable" { + require.Equal(t, phase, ticketPhase(t, s, ticket.Sig), "gap repair must not reopen terminal tickets") + } + f.mine(sent[1], true) + tickRecovery(t, s, c, key) + next := nextRecoveryTicket(ticket, 231) + require.NoError(t, s.StoreWinningTicket(next)) + f.mu.Lock() + f.tickets = append(f.tickets, next) + f.mu.Unlock() + tickRecovery(t, s, c, key) + sent = f.transactions() + require.Len(t, sent, 3) + require.EqualValues(t, 1, sent[2].Nonce()) + f.mine(sent[2], true) + tickRecovery(t, s, c, key) + require.Equal(t, "confirmed", ticketPhase(t, s, next.Sig), "queued work must actually settle after the gap is repaired") + }) + } +} + +func TestClosedTicketRepairsSupersededHigherNonce(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 232) + // The original send lost its response; a fresh attempt uses nonce one, as + // required by the retry policy. Either attempt can later win. + f.loseNextResponse = true + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + tickRecovery(t, s, c, key) + s = reopenRecoveryStore(t, s) + sent := f.transactions() + require.Len(t, sent, 2) + require.EqualValues(t, 0, sent[0].Nonce()) + require.EqualValues(t, 1, sent[1].Nonce()) + f.mine(sent[0], true) + require.Empty(t, ReconcileSubmitted(t.Context(), s, c)) + f.mu.Lock() + delete(f.pool, 1) + f.finalizedNonce = 1 + f.mu.Unlock() + // Crash after replay submission but before recording its acknowledgment. + _, err := s.db.Exec(`CREATE TRIGGER reject_gap_ack BEFORE UPDATE OF acknowledged ON redemption_transactions BEGIN SELECT RAISE(ABORT,'injected acknowledgment failure'); END`) + require.NoError(t, err) + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + s = reopenRecoveryStore(t, s) + var interrupted string + require.NoError(t, s.db.QueryRow("SELECT phase FROM redemption_transactions WHERE hash=?", sent[1].Hash().Hex()).Scan(&interrupted)) + require.Equal(t, "superseded", interrupted, "the recorded historical outcome must survive interrupted replay") + _, err = s.db.Exec("DROP TRIGGER reject_gap_ack") + require.NoError(t, err) + f.mu.Lock() + delete(f.pool, 1) + f.mu.Unlock() + tickRecovery(t, s, c, key) // No eligible tickets remain to trigger allocation. + sent = f.transactions() + require.Len(t, sent, 4) + require.Equal(t, sent[1].Hash(), sent[3].Hash()) + require.Equal(t, sent[1].Hash(), sent[2].Hash()) + require.Equal(t, "confirmed", ticketPhase(t, s, ticket.Sig)) + var current, phase string + require.NoError(t, s.db.QueryRow("SELECT tx_hash FROM winning_tickets WHERE sig=?", ticket.Sig).Scan(¤t)) + require.Equal(t, sent[0].Hash().Hex(), current) + require.NoError(t, s.db.QueryRow("SELECT phase FROM redemption_transactions WHERE hash=?", sent[1].Hash().Hex()).Scan(&phase)) + require.Equal(t, "superseded", phase) + f.mine(sent[2], false) // Already-used redemption still fills its nonce slot. + f.mu.Lock() + f.finalizedNonce = 2 + f.mu.Unlock() + next := nextRecoveryTicket(ticket, 233) + require.NoError(t, s.StoreWinningTicket(next)) + f.mu.Lock() + f.tickets = append(f.tickets, next) + f.mu.Unlock() + tickRecovery(t, s, c, key) + sent = f.transactions() + require.EqualValues(t, 2, sent[len(sent)-1].Nonce()) + f.mine(sent[len(sent)-1], true) + tickRecovery(t, s, c, key) + require.Equal(t, "confirmed", ticketPhase(t, s, next.Sig)) +} + +func TestFinalizedExternalNonceRetiresOnlyMissingTransactions(t *testing.T) { + for _, observation := range []string{"missing", "unfinalized", "receipt-error", "nonce-error", "nonce-null"} { + t.Run(observation, func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 234) + tickRecovery(t, s, c, key) + s = reopenRecoveryStore(t, s) + first := f.transactions()[0] + require.NoError(t, s.setTicketPhase(t.Context(), ticket.Sig, "expired")) + // Eviction alone retains liability; a canonical finalized account + // observation must prove another transaction consumed this nonce. + f.mu.Lock() + delete(f.pool, 0) + f.mu.Unlock() + require.Empty(t, ReconcileSubmitted(t.Context(), s, c)) + next := nextRecoveryTicket(ticket, 235) + next.CreationRound = 5 + require.ErrorIs(t, s.storeWinningTickets(t.Context(), ticket.PayerAddress, next.RecipientRandHash, 5, big.NewInt(1000), []*pm.SignedTicket{next}), pm.ErrPayerUnavailable) + f.mu.Lock() + f.finalizedNonce = 1 + f.finalizedNonceReads = 0 + f.receiptError = observation == "receipt-error" + f.finalizedNonceError = observation == "nonce-error" + f.finalizedNonceNull = observation == "nonce-null" + if observation == "unfinalized" { + f.finalized = false + f.mineLocked(first, false) + } + f.mu.Unlock() + failures := ReconcileSubmitted(t.Context(), s, c) + require.Equal(t, observation == "receipt-error" || observation == "nonce-error" || observation == "nonce-null", len(failures) > 0) + items, err := s.SubmittedRedemptions() + require.NoError(t, err) + if observation == "missing" { + require.Empty(t, items) + require.Equal(t, "expired", ticketPhase(t, s, ticket.Sig)) + require.NoError(t, s.storeWinningTickets(t.Context(), ticket.PayerAddress, next.RecipientRandHash, 5, big.NewInt(1000), []*pm.SignedTicket{next})) + oldEpoch := nextRecoveryTicket(ticket, 236) + oldEpoch.RecipientRandHash = ticket.RecipientRandHash + require.ErrorIs(t, s.storeWinningTickets(t.Context(), ticket.PayerAddress, ticket.RecipientRandHash, 0, big.NewInt(10000), []*pm.SignedTicket{oldEpoch}), ErrInvalidPayment) + } else { + require.Len(t, items, 1, "uncertain or unfinalized receipts retain liability") + } + f.mu.Lock() + reads := f.finalizedNonceReads + f.mu.Unlock() + require.LessOrEqual(t, reads, 1, "read finalized account state once per sender") + }) + } +} + +func TestFinalizedExternalNoncePreservesHigherAttempt(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 237) + f.loseNextResponse = true + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + tickRecovery(t, s, c, key) + s = reopenRecoveryStore(t, s) + f.mu.Lock() + delete(f.pool, 0) + f.finalizedNonce = 1 + f.finalizedNonceReads = 0 + f.mu.Unlock() + require.Empty(t, ReconcileSubmitted(t.Context(), s, c)) + items, err := s.SubmittedRedemptions() + require.NoError(t, err) + require.Len(t, items, 1) + require.Equal(t, f.transactions()[1].Hash(), items[0].Hash) + require.Equal(t, "submitted", ticketPhase(t, s, ticket.Sig)) + f.mu.Lock() + reads := f.finalizedNonceReads + f.mu.Unlock() + require.Equal(t, 1, reads) + next := nextRecoveryTicket(ticket, 238) + next.CreationRound = 5 + require.ErrorIs(t, s.storeWinningTickets(t.Context(), ticket.PayerAddress, next.RecipientRandHash, 5, big.NewInt(1000), []*pm.SignedTicket{next}), pm.ErrPayerUnavailable) + // Finalized state advances beyond both attempts while the pending endpoint + // stays stale. New signing must use that observed count, not the saved floor. + f.mu.Lock() + delete(f.pool, 1) + f.finalizedNonce = 3 + f.staleNonce = true + f.mu.Unlock() + tickRecovery(t, s, c, key) + sent := f.transactions() + require.Len(t, sent, 3) + require.EqualValues(t, 3, sent[2].Nonce()) + f.mine(sent[2], true) + tickRecovery(t, s, c, key) + require.Equal(t, "confirmed", ticketPhase(t, s, ticket.Sig)) +} + +func TestFinalizedNonceAdvancesPastResolvedHistory(t *testing.T) { + for _, tc := range []struct { + acknowledged bool + pending, finalized uint64 + }{{false, 0, 1}, {true, 0, 3}, {true, 1, 3}} { + t.Run(fmt.Sprintf("acknowledged=%t/pending=%d", tc.acknowledged, tc.pending), func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 239) + f.loseNextResponse = !tc.acknowledged + require.Equal(t, !tc.acknowledged, len(ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) > 0) + s = reopenRecoveryStore(t, s) + if tc.acknowledged { + f.mine(f.transactions()[0], true) + require.Empty(t, ReconcileSubmitted(t.Context(), s, c)) + require.NoError(t, s.StoreWinningTicket(nextRecoveryTicket(ticket, 240))) + } + f.mu.Lock() + delete(f.pool, 0) + f.finalizedNonce = tc.finalized + f.staleNonce = true + f.staleNonceValue = tc.pending + f.mu.Unlock() + tickRecovery(t, s, c, key) + sent := f.transactions() + require.Len(t, sent, 2) + require.Equal(t, tc.finalized, sent[1].Nonce(), "local history must not hide newer finalized account state") + }) + } +} + +func nextRecoveryTicket(ticket *pm.SignedTicket, id byte) *pm.SignedTicket { + next, body := *ticket, *ticket.Ticket + next.Ticket = &body + next.Sig = []byte{id, 2, 3} + next.RecipientRandHash = crypto.Keccak256Hash([]byte{id}) + return &next +} diff --git a/orchestrator/redemption_recovery.go b/orchestrator/redemption_recovery.go new file mode 100644 index 0000000..3f449ba --- /dev/null +++ b/orchestrator/redemption_recovery.go @@ -0,0 +1,374 @@ +// Recovery adapters for the go-livepeer port at +// 0ff08af1bb4cb268f428d619cc968b5eaee0d093. These bridge node's durable store +// and finalized settlement to the upstream nonce/send/wait/replace lifecycle. +package orchestrator + +import ( + "context" + "database/sql" + "errors" + "math/big" + "sync" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/livepeer/node/eth" +) + +type RedemptionOptions struct { + Timeout time.Duration + MaxReplacements int +} + +var defaultRedemptionOptions = RedemptionOptions{Timeout: 5 * time.Minute, MaxReplacements: 1} + +type redemptionRecovery struct { + store *RedeemerDB + chain eth.PaymentChain + key *eth.Key + nonces *eth.NonceManager + manager *eth.TransactionManager + mu sync.Mutex + tracking map[common.Hash]bool + pendingErrors []error + done chan struct{} + closeOnce sync.Once + stopParent func() bool +} + +// StartRedemptionRecovery binds long inclusion waits to the service lifetime, +// independently of the bounded heartbeat contexts used to schedule tickets. +func StartRedemptionRecovery(ctx context.Context, store *RedeemerDB, chain eth.PaymentChain, key *eth.Key, options RedemptionOptions) error { + if store == nil || key == nil || chain.Contracts == nil || chain.Contracts.RPC == nil { + return errors.New("redemption is not configured") + } + if options.Timeout <= 0 || options.MaxReplacements < 0 { + return errors.New("invalid redemption recovery options") + } + _, err := store.recoveryFor(ctx, chain, key, options) + return err +} +func (s *RedeemerDB) recoveryFor(ctx context.Context, chain eth.PaymentChain, key *eth.Key, options RedemptionOptions) (*redemptionRecovery, error) { + s.recoveryMu.Lock() + defer s.recoveryMu.Unlock() + if s.recovery != nil { + return s.recovery, nil + } + r := &redemptionRecovery{store: s, chain: chain, key: key, tracking: make(map[common.Hash]bool), done: make(chan struct{})} + r.nonces = eth.NewNonceManager(r) + floor, err := s.SavedNonceFloor(ctx, key.Address()) + if err != nil { + return nil, err + } + r.nonces.Restore(key.Address(), floor) + r.manager = eth.NewTransactionManager(r, r, key, options.Timeout, options.MaxReplacements) + r.manager.Restore = r.restore + r.manager.BeforeReplacement = r.prepareReplacement + r.manager.Completed = r.completed + r.stopParent = context.AfterFunc(ctx, r.manager.Stop) + go func() { defer close(r.done); r.manager.Start() }() + s.recovery = r + return r, nil +} +func (r *redemptionRecovery) close() { + r.closeOnce.Do(func() { r.stopParent(); r.manager.Stop(); <-r.done }) +} +func (r *redemptionRecovery) MaxGasPrice() *big.Int { return r.chain.Contracts.MaxFeePerGas } + +// Next holds the upstream account lock while reading both sources. Receipt +// reconciliation and a separate CLI process can acknowledge sends after this +// worker starts, so the startup cursor alone cannot cover stale pending reads. +func (r *redemptionRecovery) PendingNonceAt(ctx context.Context, address common.Address) (uint64, error) { + pending, err := r.chain.Contracts.RPC.PendingNonceAt(ctx, address) + if err != nil { + return 0, err + } + floor, err := r.store.SavedNonceFloor(ctx, address) + if err != nil { + return 0, err + } + return r.repairNonceGap(ctx, address, pending, floor) +} + +func (r *redemptionRecovery) CodeAt(ctx context.Context, a common.Address, n *big.Int) ([]byte, error) { + return r.chain.Contracts.RPC.CodeAt(ctx, a, n) +} +func (r *redemptionRecovery) TransactionByHash(ctx context.Context, h common.Hash) (*types.Transaction, bool, error) { + return r.chain.Contracts.RPC.TransactionByHash(ctx, h) +} +func (r *redemptionRecovery) TransactionReceipt(ctx context.Context, h common.Hash) (*types.Receipt, error) { + return r.chain.InclusionReceipt(ctx, h) +} + +func (r *redemptionRecovery) SendTransaction(ctx context.Context, tx *types.Transaction) error { + var sig []byte + var root string + var previous sql.NullString + if err := r.store.db.QueryRowContext(ctx, "SELECT sig,root_hash,replaces_hash FROM redemption_transactions WHERE hash=?", tx.Hash().Hex()).Scan(&sig, &root, &previous); err != nil { + return err + } + r.mu.Lock() + r.tracking[common.HexToHash(root)] = true + r.mu.Unlock() + identity, err := signedIdentity(tx) + if err == nil { + err = broadcastRedemption(ctx, r.store, r.chain.Contracts, sig, identity) + } + if err != nil && !previous.Valid { + r.mu.Lock() + delete(r.tracking, common.HexToHash(root)) + r.mu.Unlock() + } + return err +} +func (r *redemptionRecovery) prepareReplacement(ctx context.Context, previous, next *types.Transaction) error { + identity, err := signedIdentity(next) + if err != nil { + return err + } + if identity.From != r.key.Address() { + return errors.New("replacement transaction account mismatch") + } + var sig []byte + if err := r.store.db.QueryRowContext(ctx, "SELECT sig FROM redemption_transactions WHERE hash=?", previous.Hash().Hex()).Scan(&sig); err != nil { + return err + } + return r.store.recordTransaction(ctx, sig, identity, previous.Hash().Hex()) +} +func (r *redemptionRecovery) restore(ctx context.Context, tx *types.Transaction) (eth.TransactionRecovery, error) { + var root string + var phase, identityPhase string + err := r.store.db.QueryRowContext(ctx, `SELECT t.root_hash,a.phase,t.phase FROM redemption_transactions t JOIN redemption_attempts a ON a.sig=t.sig WHERE t.hash=?`, tx.Hash().Hex()).Scan(&root, &phase, &identityPhase) + if err != nil { + return eth.TransactionRecovery{}, err + } + if terminalRedemption(phase) || identityPhase == "superseded" { + return eth.TransactionRecovery{OriginHash: common.HexToHash(root)}, context.Canceled + } + rows, err := r.store.db.QueryContext(ctx, `SELECT t.hash,t.raw_transaction,COALESCE(t.broadcast_at,t.prepared_at),w.tx_hash=t.hash FROM redemption_transactions t JOIN winning_tickets w ON w.sig=t.sig WHERE t.root_hash=? ORDER BY t.rowid`, root) + if err != nil { + return eth.TransactionRecovery{}, err + } + defer rows.Close() + state := eth.TransactionRecovery{OriginHash: common.HexToHash(root), Replacements: -1} + for rows.Next() { + var hash, at string + var raw []byte + var current bool + if err := rows.Scan(&hash, &raw, &at, ¤t); err != nil { + return state, err + } + latest := new(types.Transaction) + if err := latest.UnmarshalBinary(raw); err != nil { + return state, errors.New("invalid stored redemption transaction") + } + state.Hashes = append(state.Hashes, common.HexToHash(hash)) + state.Replacements++ + if current { + state.Transaction = latest + state.SentAt, err = time.Parse(time.RFC3339Nano, at) + if err != nil { + return state, errors.New("invalid stored redemption timestamp") + } + } + } + if err := rows.Err(); err != nil { + return state, err + } + if state.Transaction == nil { + return state, context.Canceled + } + return state, nil +} +func (r *redemptionRecovery) completed(ctx context.Context, result *eth.TransactionReceipt) { + defer func() { r.mu.Lock(); delete(r.tracking, result.OriginHash()); r.mu.Unlock() }() + if err := r.recordResult(ctx, result); err != nil && !errors.Is(err, context.Canceled) { + r.mu.Lock() + r.pendingErrors = append(r.pendingErrors, err) + r.mu.Unlock() + } +} +func (r *redemptionRecovery) recordResult(ctx context.Context, result *eth.TransactionReceipt) error { + root := result.OriginHash().Hex() + if errors.Is(result.Error(), context.Canceled) || errors.Is(result.Error(), eth.ErrReceiptUnavailable) { + return nil + } + phase := "mined" + var message any + if result.Error() != nil && !errors.Is(result.Error(), eth.ErrReplacingMinedTx) { + phase = "retryable" + message = result.Error().Error() + } + tx, err := r.store.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if phase == "mined" && result.TxHash != (common.Hash{}) { + _, err = tx.ExecContext(ctx, `UPDATE redemption_transactions SET phase='mined',error=NULL,acknowledged=1 WHERE hash=? AND phase NOT IN ('confirmed','reverted','superseded')`, result.TxHash.Hex()) + } else if phase == "retryable" { + _, err = tx.ExecContext(ctx, `UPDATE redemption_transactions SET phase=?,error=? WHERE root_hash=? AND phase IN ('prepared','broadcast','sending','submitted','mined')`, phase, message, root) + } + if err != nil { + return err + } + _, err = tx.ExecContext(ctx, `UPDATE redemption_attempts SET phase=?,error=? WHERE phase IN ('prepared','broadcast','submitted','mined') AND sig IN (SELECT sig FROM redemption_transactions WHERE root_hash=?) AND EXISTS(SELECT 1 FROM winning_tickets w JOIN redemption_transactions t ON t.hash=w.tx_hash WHERE w.sig=redemption_attempts.sig AND t.root_hash=?)`, phase, message, root, root) + if err != nil { + return err + } + return tx.Commit() +} + +func terminalRedemption(phase string) bool { + return phase == "confirmed" || phase == "reverted" || phase == "used" || phase == "invalid" || phase == "expired" +} + +// resume first restores each active upstream queue job. Old 'sending' rows are +// uncertain sends, not permanent blockers. Failed groups remain receipt-visible. +func (r *redemptionRecovery) resume(ctx context.Context) []error { + failures := r.reconcileUsed(ctx) + // Repair old reservations even when no eligible ticket triggers Next. Never + // reset the upstream cursor on an ambiguous pending-nonce observation. + floor, err := r.store.SavedNonceFloor(ctx, r.key.Address()) + if err == nil && floor > 0 { + r.nonces.Lock(r.key.Address()) + _, err = r.PendingNonceAt(ctx, r.key.Address()) + r.nonces.Unlock(r.key.Address()) + } + if err != nil && !errors.Is(err, errRecoveringNonce) { + failures = append(failures, err) + } + r.mu.Lock() + failures = append(failures, r.pendingErrors...) + r.pendingErrors = nil + r.mu.Unlock() + rows, err := r.store.db.QueryContext(ctx, `SELECT t.root_hash,t.raw_transaction,t.phase FROM winning_tickets w JOIN redemption_attempts a ON a.sig=w.sig JOIN redemption_transactions t ON t.hash=w.tx_hash WHERE a.phase IN ('prepared','broadcast','submitted','mined') ORDER BY t.rowid`) + if err != nil { + return []error{err} + } + type job struct { + root common.Hash + raw []byte + phase string + } + var jobs []job + for rows.Next() { + var root string + var j job + if err := rows.Scan(&root, &j.raw, &j.phase); err != nil { + rows.Close() + return []error{err} + } + j.root = common.HexToHash(root) + jobs = append(jobs, j) + } + err = rows.Err() + rows.Close() + if err != nil { + return []error{err} + } + for _, j := range jobs { + r.mu.Lock() + active := r.tracking[j.root] + if !active { + r.tracking[j.root] = true + } + r.mu.Unlock() + if active { + continue + } + tx := new(types.Transaction) + err := tx.UnmarshalBinary(j.raw) + if err == nil { + if j.phase == "prepared" || j.phase == "broadcast" || j.phase == "sending" { + // No live sender exists for this group; recover a crash during a send. + if j.phase == "sending" { + _, err = r.store.db.ExecContext(ctx, "UPDATE redemption_transactions SET phase='broadcast' WHERE hash=? AND phase='sending'", tx.Hash().Hex()) + } + if err == nil { + err = r.manager.SendTransaction(ctx, tx) + } + if err == nil { + identity, e := signedIdentity(tx) + if e == nil && identity.Nonce < ^uint64(0) { + r.nonces.Restore(identity.From, identity.Nonce+1) + } + } + } else { + r.manager.Track(tx) + } + } + if err != nil { + failures = append(failures, err) + r.mu.Lock() + delete(r.tracking, j.root) + r.mu.Unlock() + } + } + return failures +} + +// Keep this adapter's interface narrow despite geth's broader embedded readers. +var _ ethereum.TransactionSender = (*redemptionRecovery)(nil) + +// A latest-state used ticket pauses the upstream queue. Settlement remains pinned +// to finalized state; a reorg that removes that use makes the ticket retryable. +func (r *redemptionRecovery) reconcileUsed(ctx context.Context) []error { + rows, err := r.store.db.QueryContext(ctx, `SELECT sig,phase FROM redemption_attempts a WHERE phase IN ('awaiting_finality','mined') OR (phase='reverted' AND EXISTS(SELECT 1 FROM redemption_transactions t WHERE t.sig=a.sig AND t.phase NOT IN ('confirmed','reverted','superseded')))`) + if err != nil { + return []error{err} + } + type waiting struct { + sig []byte + phase string + } + var signatures []waiting + for rows.Next() { + var item waiting + if err := rows.Scan(&item.sig, &item.phase); err != nil { + rows.Close() + return []error{err} + } + signatures = append(signatures, item) + } + err = rows.Err() + rows.Close() + if err != nil { + return []error{err} + } + var failures []error + for _, item := range signatures { + sig := item.sig + ticket, err := r.store.winningTicket(ctx, sig) + if err != nil { + failures = append(failures, err) + continue + } + used, err := r.chain.IsUsedTicket(ctx, ticket.Hash(), "latest") + if err != nil { + failures = append(failures, err) + continue + } + phase := "retryable" + if !used && item.phase != "awaiting_finality" { + continue + } + if used { + finalized, err := r.chain.IsUsedTicket(ctx, ticket.Hash(), "finalized") + if err != nil { + failures = append(failures, err) + continue + } + if !finalized { + continue + } + phase = "used" + } + if err := r.store.setTicketPhase(ctx, sig, phase); err != nil { + failures = append(failures, err) + } + } + return failures +} diff --git a/orchestrator/redemption_recovery_test.go b/orchestrator/redemption_recovery_test.go index 4457262..18baad4 100644 --- a/orchestrator/redemption_recovery_test.go +++ b/orchestrator/redemption_recovery_test.go @@ -9,147 +9,630 @@ import ( "net/http/httptest" "path/filepath" "strings" - "sync/atomic" + "sync" "testing" + "time" - ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/livepeer/node/eth" "github.com/livepeer/node/internal/test" "github.com/livepeer/node/pm" "github.com/stretchr/testify/require" ) -func TestRedemptionRecoveryBeforeAndAfterBroadcast(t *testing.T) { - for _, failure := range []string{"preparation", "prepared-crash", "uncertain-broadcast"} { - t.Run(failure, func(t *testing.T) { - keyFile, passwordPath := test.WriteFixedKeystore(t) - key, err := eth.OpenKeystoreFile(keyFile, passwordPath) - require.NoError(t, err) - path := filepath.Join(t.TempDir(), "payments.sqlite") - store, err := OpenRedeemerDB(path) +// recoveryRPC models independent clients against one account's pending pool. +// All mutable RPC state is synchronized so the same scenarios run under -race. +type recoveryRPC struct { + staleNonceValue uint64 + prepareError string + snapshotError, receiptMissingStatus bool + block, round int64 + inactive, activeError bool + activeRecipient common.Address + onSend func(*types.Transaction) + mu sync.Mutex + tickets []*pm.SignedTicket + sent []*types.Transaction + pool map[uint64]*types.Transaction + receipts map[common.Hash]*types.Receipt + used map[common.Hash]bool + finalized bool + receiptError, receiptMissing bool + staleNonce bool + finalizedNonce uint64 + finalizedNonceReads int + finalizedNonceError, finalizedNonceNull bool + rejectNext, loseNextResponse bool + mineAfter int + nonceReads int + contractCalls int + rejectSend int + nonceBarrier chan struct{} + receiptHeader *types.Header + server *httptest.Server +} + +func newRecoveryRPC(t *testing.T) *recoveryRPC { + t.Helper() + f := &recoveryRPC{pool: map[uint64]*types.Transaction{}, receipts: map[common.Hash]*types.Receipt{}, used: map[common.Hash]bool{}, finalized: true, block: 100, round: 1, receiptHeader: &types.Header{Number: big.NewInt(10), Difficulty: new(big.Int), BaseFee: big.NewInt(100)}} + f.server = httptest.NewServer(http.HandlerFunc(f.serve)) + t.Cleanup(f.server.Close) + return f +} +func (f *recoveryRPC) serve(w http.ResponseWriter, r *http.Request) { + var request struct { + ID json.RawMessage + Method string + Params []json.RawMessage + } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + http.Error(w, "bad request", 400) + return + } + f.mu.Lock() + var result any = "0x1" + var failure string + var barrier chan struct{} + switch request.Method { + case "eth_call": + f.contractCalls++ + var call struct{ Input string } + _ = json.Unmarshal(request.Params[0], &call) + selector := func(signature string) bool { + return strings.HasPrefix(call.Input, fmt.Sprintf("0x%x", crypto.Keccak256([]byte(signature))[:4])) + } + result = fmt.Sprintf("0x%064x", 1) + if selector("lastInitializedRound()") { + result = fmt.Sprintf("0x%064x", f.round) + } + if selector("isActiveTranscoder(address)") { + f.activeRecipient = common.HexToAddress(call.Input[len(call.Input)-40:]) + if f.activeError { + failure = "active lookup unavailable" + } + if f.inactive { + result = fmt.Sprintf("0x%064x", 0) + } + } + if selector("usedTickets(bytes32)") { + hash := common.HexToHash("0x" + call.Input[10:]) + used := f.used[hash] + // Finalized reads use an EIP-1898 canonical hash, not the moving latest tag. + if string(request.Params[1]) != "\"latest\"" && !f.finalized { + used = false + } + result = fmt.Sprintf("0x%064x", 0) + if used { + result = fmt.Sprintf("0x%064x", 1) + } + } + case "eth_getBlockByNumber": + var block string + _ = json.Unmarshal(request.Params[0], &block) + if block == "latest" && f.snapshotError { + failure = "snapshot unavailable" + } + header := &types.Header{Number: big.NewInt(f.block), Difficulty: new(big.Int), BaseFee: big.NewInt(100)} + if block == "0xa" { + header = f.receiptHeader + } + if block == "finalized" && !f.finalized { + header = &types.Header{Number: big.NewInt(9), Difficulty: new(big.Int), BaseFee: big.NewInt(100)} + } + result = header + case "eth_estimateGas": + failure = f.prepareError + result = "0x5208" + case "eth_maxPriorityFeePerGas": + result = "0x3" + case "eth_getTransactionCount": + nonce := f.finalizedNonce + if string(request.Params[1]) != `"pending"` { + f.finalizedNonceReads++ + result = fmt.Sprintf("0x%x", nonce) + if f.finalizedNonceError { + failure = "finalized nonce lookup unavailable" + } + if f.finalizedNonceNull { + result = nil + } + break + } + for f.pool[nonce] != nil { + nonce++ + } + if f.staleNonce { + nonce = f.staleNonceValue + } + result = fmt.Sprintf("0x%x", nonce) + f.nonceReads++ + if f.nonceBarrier != nil && f.nonceReads <= 2 { + barrier = f.nonceBarrier + if f.nonceReads == 2 { + close(barrier) + } + } + case "eth_sendRawTransaction": + var encoded string + _ = json.Unmarshal(request.Params[0], &encoded) + raw, err := hex.DecodeString(strings.TrimPrefix(encoded, "0x")) + if err != nil { + failure = "bad raw transaction" + break + } + tx := new(types.Transaction) + if err := tx.UnmarshalBinary(raw); err != nil { + failure = "bad transaction" + break + } + f.sent = append(f.sent, tx) + if f.onSend != nil { + f.onSend(tx) + } + if tx.Nonce() < f.finalizedNonce { + failure = "nonce too low" + break + } + if f.rejectNext || len(f.sent) == f.rejectSend { + f.rejectNext = false + failure = "insufficient funds" + break + } + old := f.pool[tx.Nonce()] + if old != nil && old.Hash() != tx.Hash() && (tx.GasFeeCap().Cmp(old.GasFeeCap()) <= 0 || tx.GasTipCap().Cmp(old.GasTipCap()) <= 0) { + failure = "replacement transaction underpriced" + break + } + f.pool[tx.Nonce()] = tx + result = tx.Hash().Hex() + if f.mineAfter > 0 && len(f.sent) >= f.mineAfter { + f.mineLocked(tx, true) + } + if f.loseNextResponse { + f.loseNextResponse = false + f.mu.Unlock() + http.Error(w, "lost response", 503) + return + } + case "eth_getTransactionReceipt": + var hash common.Hash + _ = json.Unmarshal(request.Params[0], &hash) + result = f.receipts[hash] + if f.receipts[hash] != nil && f.receiptMissingStatus { + encoded, _ := json.Marshal(result) + var fields map[string]any + _ = json.Unmarshal(encoded, &fields) + delete(fields, "status") + result = fields + } + if f.receiptMissing { + result = nil + } + if f.receiptError { + failure = "receipt lookup unavailable" + } + case "eth_getTransactionByHash": + var hash common.Hash + _ = json.Unmarshal(request.Params[0], &hash) + result = nil + for _, tx := range f.sent { + pending := f.pool[tx.Nonce()] + if tx.Hash() == hash && (f.receipts[hash] != nil || (pending != nil && pending.Hash() == hash)) { + encoded, _ := json.Marshal(tx) + var fields map[string]any + _ = json.Unmarshal(encoded, &fields) + if receipt := f.receipts[hash]; receipt != nil { + fields["blockNumber"] = "0xa" + fields["blockHash"] = f.receiptHeader.Hash() + } + result = fields + break + } + } + default: + failure = "unexpected RPC " + request.Method + } + f.mu.Unlock() + if barrier != nil { + select { + case <-barrier: + case <-r.Context().Done(): + return + } + } + response := map[string]any{"jsonrpc": "2.0", "id": request.ID, "result": result} + if failure != "" { + delete(response, "result") + response["error"] = map[string]any{"code": -32000, "message": failure} + } + _ = json.NewEncoder(w).Encode(response) +} +func (f *recoveryRPC) mineLocked(tx *types.Transaction, success bool) { + status := uint64(0) + if success { + status = 1 + } + f.receipts[tx.Hash()] = &types.Receipt{TxHash: tx.Hash(), Status: status, BlockNumber: big.NewInt(10), BlockHash: f.receiptHeader.Hash(), Logs: []*types.Log{}} + if success { + for _, ticket := range f.tickets { + if strings.Contains(hex.EncodeToString(tx.Data()), hex.EncodeToString(ticket.RecipientRandHash[:])) { + f.used[ticket.Hash()] = true + } + } + } +} +func (f *recoveryRPC) transactions() []*types.Transaction { + f.mu.Lock() + defer f.mu.Unlock() + return append([]*types.Transaction(nil), f.sent...) +} +func (f *recoveryRPC) mine(tx *types.Transaction, success bool) { + f.mu.Lock() + defer f.mu.Unlock() + f.mineLocked(tx, success) +} +func (f *recoveryRPC) chain(t *testing.T) eth.PaymentChain { + t.Helper() + rpc, err := eth.OpenRPC(f.server.URL) + require.NoError(t, err) + t.Cleanup(rpc.Close) + c, err := eth.NewContracts(rpc, common.HexToAddress("0x1000")) + require.NoError(t, err) + return eth.PaymentChain{Contracts: c} +} +func recoveryKey(t *testing.T) *eth.Key { + t.Helper() + path, password := test.WriteFixedKeystore(t) + key, err := eth.OpenKeystoreFile(path, password) + require.NoError(t, err) + return key +} +func (f *recoveryRPC) store(t *testing.T, id byte) (*RedeemerDB, *pm.SignedTicket) { + t.Helper() + store, err := OpenRedeemerDB(filepath.Join(t.TempDir(), "payments.sqlite")) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, store.Close()) }) + ticket := &pm.SignedTicket{Ticket: &pm.Ticket{PayerAddress: common.HexToAddress("0x1234"), Recipient: common.HexToAddress("0x5678"), FaceValue: big.NewInt(1000), WinProb: big.NewInt(100), TicketNonce: 1, RecipientRandHash: crypto.Keccak256Hash([]byte{id}), CreationRound: 1, ParamsExpirationBlock: big.NewInt(10)}, Sig: []byte{id, 2, 3}, RecipientRand: big.NewInt(int64(id))} + require.NoError(t, store.StoreWinningTicket(ticket)) + f.mu.Lock() + f.tickets = append(f.tickets, ticket) + f.mu.Unlock() + return store, ticket +} +func ticketPhase(t *testing.T, s *RedeemerDB, sig []byte) string { + t.Helper() + var phase string + require.NoError(t, s.db.QueryRow("SELECT phase FROM redemption_attempts WHERE sig=?", sig).Scan(&phase)) + return phase +} +func tickRecovery(t *testing.T, s *RedeemerDB, c eth.PaymentChain, k *eth.Key) { + t.Helper() + require.Empty(t, ProcessRedemptions(t.Context(), s, c, k, big.NewInt(1))) +} + +// newRecoveryTest shares setup without starting recovery or changing RPC behavior. +func newRecoveryTest(t *testing.T, id byte) (*recoveryRPC, *RedeemerDB, *pm.SignedTicket, eth.PaymentChain, *eth.Key) { + t.Helper() + f := newRecoveryRPC(t) + store, ticket := f.store(t, id) + return f, store, ticket, f.chain(t), recoveryKey(t) +} + +func TestSharedWalletRedemptionRecovery(t *testing.T) { + f := newRecoveryRPC(t) + f.nonceBarrier = make(chan struct{}) + key := recoveryKey(t) + a, ta := f.store(t, 1) + b, tb := f.store(t, 2) + ca, cb := f.chain(t), f.chain(t) + results := make(chan []error, 2) + go func() { results <- ProcessRedemptions(t.Context(), a, ca, key, big.NewInt(1)) }() + go func() { results <- ProcessRedemptions(t.Context(), b, cb, key, big.NewInt(1)) }() + first, second := <-results, <-results + require.Equal(t, 1, len(first)+len(second)) + sent := f.transactions() + require.Len(t, sent, 2) + require.EqualValues(t, 0, sent[0].Nonce()) + require.EqualValues(t, 0, sent[1].Nonce()) + require.NotEqual(t, sent[0].Hash(), sent[1].Hash()) + // The losing process discovers pending nonce 1 and retries its own ticket. + loser, lc, lt := a, ca, ta + if ticketPhase(t, b, tb.Sig) == "retryable" { + loser, lc, lt = b, cb, tb + } + floor, err := loser.SavedNonceFloor(t.Context(), key.Address()) + require.NoError(t, err) + require.Zero(t, floor) + tickRecovery(t, loser, lc, key) + sent = f.transactions() + require.Len(t, sent, 3) + require.EqualValues(t, 1, sent[2].Nonce()) + require.NotEqual(t, "retryable", ticketPhase(t, loser, lt.Sig)) + f.mine(sent[0], true) + f.mine(sent[2], true) + tickRecovery(t, a, ca, key) + tickRecovery(t, b, cb, key) + require.Equal(t, "confirmed", ticketPhase(t, a, ta.Sig)) + require.Equal(t, "confirmed", ticketPhase(t, b, tb.Sig)) + history, err := loser.Redemptions() + require.NoError(t, err) + require.Len(t, history, 2) +} + +func TestRejectedRedemptionDoesNotSkipNonce(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 3) + f.rejectNext = true + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + require.Equal(t, "retryable", ticketPhase(t, s, ticket.Sig)) + tickRecovery(t, s, c, key) + sent := f.transactions() + require.Len(t, sent, 2) + require.EqualValues(t, 0, sent[1].Nonce()) + require.Equal(t, sent[0].Hash(), sent[1].Hash()) +} + +func TestRedemptionReplacementAndFinality(t *testing.T) { + for _, winner := range []int{0, 1} { + t.Run(fmt.Sprintf("winner=%d", winner), func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 4) + f.finalized = false + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, RedemptionOptions{Timeout: 30 * time.Millisecond, MaxReplacements: 1})) + tickRecovery(t, s, c, key) + require.Eventually(t, func() bool { return len(f.transactions()) == 2 }, time.Second, 5*time.Millisecond) + sent := f.transactions() + require.Equal(t, sent[0].Nonce(), sent[1].Nonce()) + require.Equal(t, sent[0].Data(), sent[1].Data()) + require.Equal(t, big.NewInt(224), sent[1].GasFeeCap()) + require.Equal(t, big.NewInt(4), sent[1].GasTipCap()) + f.mine(sent[winner], true) + tickRecovery(t, s, c, key) + require.NotEqual(t, "confirmed", ticketPhase(t, s, ticket.Sig)) + require.Eventually(t, func() bool { return ticketPhase(t, s, ticket.Sig) == "mined" }, time.Second, 5*time.Millisecond) + f.mu.Lock() + f.finalized = true + f.mu.Unlock() + tickRecovery(t, s, c, key) + require.Equal(t, "confirmed", ticketPhase(t, s, ticket.Sig)) + var hash string + require.NoError(t, s.db.QueryRow("SELECT tx_hash FROM winning_tickets WHERE sig=?", ticket.Sig).Scan(&hash)) + require.Equal(t, sent[winner].Hash().Hex(), hash) + items, err := s.Redemptions() require.NoError(t, err) - defer func() { _ = store.Close() }() - ticket := &pm.SignedTicket{Ticket: &pm.Ticket{PayerAddress: ethcommon.HexToAddress("0x1234"), Recipient: ethcommon.HexToAddress("0x5678"), FaceValue: big.NewInt(10), WinProb: big.NewInt(100), TicketNonce: 1, RecipientRandHash: ethcommon.HexToHash("0xabcd"), CreationRound: 1, ParamsExpirationBlock: big.NewInt(10)}, Sig: []byte{1, 2, 3}, RecipientRand: big.NewInt(4)} - require.NoError(t, store.StoreWinningTicket(ticket)) - var fail atomic.Bool - fail.Store(true) - var snapshotUnavailable, validReceipt atomic.Bool - var broadcasts atomic.Int32 - var firstRaw string - receiptHeader := &types.Header{Number: big.NewInt(1), Difficulty: new(big.Int), BaseFee: big.NewInt(1)} - server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var req struct { - Method string - Params []json.RawMessage + require.Len(t, items, 2) + for i, tx := range sent { + require.Equal(t, tx.Hash().Hex(), items[i].Hash) + require.Error(t, RetryRedemption(t.Context(), s, c.Contracts, tx.Hash())) + } + require.Len(t, f.transactions(), 2) + }) + } +} + +func TestRedemptionReplacementFailuresRemainQueued(t *testing.T) { + for _, scenario := range []string{"limit", "ceiling", "rejection", "persistence"} { + t.Run(scenario, func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 6) + options := RedemptionOptions{Timeout: 30 * time.Millisecond, MaxReplacements: 1} + if scenario == "limit" { + options.MaxReplacements = 0 + } + if scenario == "rejection" { + f.rejectSend = 2 + options.MaxReplacements = 3 + } + if scenario == "ceiling" { + c.Contracts.MaxFeePerGas = big.NewInt(210) + } + if scenario == "persistence" { + _, err := s.db.Exec(`CREATE TRIGGER reject_replacement BEFORE INSERT ON redemption_transactions WHEN NEW.replaces_hash IS NOT NULL BEGIN SELECT RAISE(ABORT,'storage failure'); END`) + require.NoError(t, err) + } + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, options)) + tickRecovery(t, s, c, key) + require.Eventually(t, func() bool { return ticketPhase(t, s, ticket.Sig) == "retryable" }, time.Second, 5*time.Millisecond) + expected := 1 + if scenario == "rejection" { + expected = 2 + } + require.Len(t, f.transactions(), expected) + if scenario == "rejection" { + var acknowledged bool + require.NoError(t, s.db.QueryRow("SELECT acknowledged FROM redemption_transactions WHERE replaces_hash IS NOT NULL").Scan(&acknowledged)) + require.False(t, acknowledged, "the rejected replacement must stop the remaining replacement budget") + } + // A timeout/rejected replacement does not settle the still-live ticket. + var redeemed *string + require.NoError(t, s.db.QueryRow("SELECT redeemed_at FROM winning_tickets WHERE sig=?", ticket.Sig).Scan(&redeemed)) + require.Nil(t, redeemed) + }) + } +} + +func TestLostResponseUsesFinalizedTicketGuard(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 7) + f.loseNextResponse = true + f.finalized = false + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + sent := f.transactions() + f.mine(sent[0], true) + tickRecovery(t, s, c, key) + require.Equal(t, "awaiting_finality", ticketPhase(t, s, ticket.Sig)) + require.Len(t, f.transactions(), 1) + f.mu.Lock() + f.finalized = true + f.mu.Unlock() + tickRecovery(t, s, c, key) + require.Equal(t, "confirmed", ticketPhase(t, s, ticket.Sig)) + require.Len(t, f.transactions(), 1) + floor, err := s.SavedNonceFloor(t.Context(), key.Address()) + require.NoError(t, err) + require.EqualValues(t, 1, floor, "a finalized receipt proves acceptance even when the send response was lost") +} + +func TestUsedTicketGuardRecoversAfterReorg(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 8) + f.finalized = false + f.used[ticket.Hash()] = true + tickRecovery(t, s, c, key) + require.Equal(t, "awaiting_finality", ticketPhase(t, s, ticket.Sig)) + require.Empty(t, f.transactions()) + f.mu.Lock() + delete(f.used, ticket.Hash()) + f.mu.Unlock() + tickRecovery(t, s, c, key) + require.Len(t, f.transactions(), 1) +} + +func reopenRecoveryStore(t *testing.T, s *RedeemerDB) *RedeemerDB { + t.Helper() + var seq int + var name, path string + require.NoError(t, s.db.QueryRow("PRAGMA database_list").Scan(&seq, &name, &path)) + require.NoError(t, s.Close()) + next, err := OpenRedeemerDB(path) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, next.Close()) }) + return next +} + +func TestRedemptionRestartRestoresTransactionCursor(t *testing.T) { + for _, boundary := range []string{"prepared", "sending", "submitted", "replacement-prepared", "replacement-submitted"} { + t.Run(boundary, func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 10) + if boundary == "prepared" { + prepared, err := c.PrepareRedemption(t.Context(), key, big.NewInt(1), redeemTicket(ticket)) + require.NoError(t, err) + require.NoError(t, s.recordPrepared(t.Context(), ticket, prepared)) + } else { + options := RedemptionOptions{Timeout: time.Hour, MaxReplacements: 1} + if boundary == "replacement-submitted" { + options.Timeout = 100 * time.Millisecond } - require.NoError(t, json.NewDecoder(r.Body).Decode(&req)) - var result any = "0x1" - switch req.Method { - case "eth_call": - // Reuse the 32-byte encoding of 1 for calls returning an address, round, hash, or bool. - result = fmt.Sprintf("0x%064x", 1) - case "eth_getBlockByNumber": - if snapshotUnavailable.Load() && string(req.Params[0]) == `"latest"` { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - result = &types.Header{Number: big.NewInt(10), Difficulty: new(big.Int), BaseFee: big.NewInt(1)} - if string(req.Params[0]) == `"0x1"` { - result = receiptHeader - } - case "eth_estimateGas": - if failure == "preparation" && fail.Load() { - http.Error(w, "unavailable", http.StatusServiceUnavailable) - return - } - result = "0x5208" - case "eth_getTransactionReceipt": - result = map[string]any{"transactionHash": req.Params[0], "blockNumber": "0x1", "blockHash": receiptHeader.Hash(), - "cumulativeGasUsed": "0x0", "gasUsed": "0x0", "logsBloom": types.Bloom{}, "logs": []any{}} - if validReceipt.Load() { - result.(map[string]any)["status"] = "0x1" - } - case "eth_maxPriorityFeePerGas", "eth_getTransactionCount": - case "eth_sendRawTransaction": - broadcasts.Add(1) - var encoded string - require.NoError(t, json.Unmarshal(req.Params[0], &encoded)) - if firstRaw != "" { - require.Equal(t, firstRaw, encoded, "recovery must reuse the exact transaction") - } - firstRaw = encoded - if failure == "uncertain-broadcast" && fail.Load() { - http.Error(w, "lost response", http.StatusServiceUnavailable) - return - } - raw, err := hex.DecodeString(strings.TrimPrefix(encoded, "0x")) - require.NoError(t, err) - var transaction types.Transaction - require.NoError(t, transaction.UnmarshalBinary(raw)) - result = transaction.Hash().Hex() - default: - t.Errorf("unexpected RPC %s", req.Method) + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, options)) + tickRecovery(t, s, c, key) + if boundary == "replacement-submitted" { + require.Eventually(t, func() bool { + var count int + err := s.db.QueryRow("SELECT count(*) FROM redemption_transactions WHERE acknowledged=1").Scan(&count) + return err == nil && count == 2 + }, time.Second, time.Millisecond) } - _ = json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "result": result}) - })) - defer server.Close() - rpc, err := eth.OpenRPC(server.URL) - require.NoError(t, err) - contracts, err := eth.OpenContracts(rpc, ethcommon.HexToAddress("0x1000").Hex()) - require.NoError(t, err) - chain := eth.PaymentChain{Contracts: contracts} + } + // Stop the old runtime before editing a durable crash-boundary fixture. + s = reopenRecoveryStore(t, s) + if boundary == "sending" { + _, err := s.db.Exec("UPDATE redemption_attempts SET phase='broadcast'") + require.NoError(t, err) + _, err = s.db.Exec("UPDATE redemption_transactions SET phase='sending',acknowledged=0") + require.NoError(t, err) + } + if boundary == "replacement-prepared" { + first := f.transactions()[0] + replacement := recoveryReplacement(t, key, first) + identity, err := signedIdentity(replacement) + require.NoError(t, err) + require.NoError(t, s.recordTransaction(t.Context(), ticket.Sig, identity, first.Hash().Hex())) + } + expectedSends := 1 + if boundary == "sending" || strings.HasPrefix(boundary, "replacement-") { + expectedSends = 2 + } + var originalDeadline string + if boundary == "sending" { + require.NoError(t, s.db.QueryRow("SELECT broadcast_at FROM redemption_transactions").Scan(&originalDeadline)) + } + options := RedemptionOptions{Timeout: time.Hour, MaxReplacements: 1} + if boundary == "replacement-submitted" { + options.Timeout = time.Nanosecond + } + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, options)) + tickRecovery(t, s, c, key) + require.Len(t, f.transactions(), expectedSends) + if boundary == "replacement-submitted" { + require.Eventually(t, func() bool { return ticketPhase(t, s, ticket.Sig) == "retryable" }, time.Second, time.Millisecond) + require.Len(t, f.transactions(), 2, "restart must retain the spent replacement budget") + } + if boundary == "sending" { + var at string + require.NoError(t, s.db.QueryRow("SELECT broadcast_at FROM redemption_transactions").Scan(&at)) + require.Equal(t, originalDeadline, at, "restarting an uncertain send must preserve its timeout") + sent := f.transactions() + require.Equal(t, sent[0].Hash(), sent[1].Hash()) + } + }) + } +} + +func TestRedemptionRecoveryBeforeAndAfterBroadcast(t *testing.T) { + for _, failure := range []string{"preparation", "prepared-crash", "uncertain-broadcast"} { + t.Run(failure, func(t *testing.T) { + f, store, ticket, chain, key := newRecoveryTest(t, 1) + f.finalizedNonce = 1 + if failure == "preparation" { + f.prepareError = "preparation unavailable" + } + f.loseNextResponse = failure == "uncertain-broadcast" snapshot, err := chain.Snapshot(t.Context()) require.NoError(t, err) + var prepared eth.SignedTransaction if failure == "prepared-crash" { - prepared, err := chain.PrepareRedemption(t.Context(), key, big.NewInt(1), eth.RedeemTicket{Recipient: ticket.Recipient, Sender: ticket.PayerAddress, FaceValue: ticket.FaceValue, WinProb: ticket.WinProb, SenderNonce: ticket.TicketNonce, RecipientRandHash: ticket.RecipientRandHash, AuxData: ticket.AuxData(), Signature: ticket.Sig, RecipientRand: ticket.RecipientRand}) + prepared, err = chain.PrepareRedemption(t.Context(), key, big.NewInt(1), redeemTicket(ticket)) require.NoError(t, err) - firstRaw = "0x" + hex.EncodeToString(prepared.Raw) require.NoError(t, store.recordPrepared(t.Context(), ticket, prepared)) } else { require.NotEmpty(t, RedeemPending(t.Context(), store, chain, key, big.NewInt(1), snapshot)) } - require.NoError(t, store.Close()) - store, err = OpenRedeemerDB(path) - require.NoError(t, err) - fail.Store(false) + store = reopenRecoveryStore(t, store) + f.mu.Lock() + f.prepareError = "" + f.staleNonce = true + f.mu.Unlock() restartKey := key if failure == "prepared-crash" { restartKey = paymentTestKey(t) } require.Empty(t, RedeemPending(t.Context(), store, chain, restartKey, big.NewInt(1), snapshot)) - require.Equal(t, int32(1), broadcasts.Load(), "uncertain sends cannot retry automatically") - items, err := store.Redemptions() - require.NoError(t, err) - require.Len(t, items, 1) + sent := f.transactions() + expectedSends := 1 if failure == "uncertain-broadcast" { - require.Equal(t, "broadcast", items[0].Phase) - require.NoError(t, RetryRedemption(t.Context(), store, contracts, ethcommon.HexToHash(items[0].Hash))) - require.Equal(t, int32(2), broadcasts.Load()) + expectedSends = 2 } + require.Len(t, sent, expectedSends, "retryable sends recover automatically") + if failure == "prepared-crash" { + raw, err := sent[0].MarshalBinary() + require.NoError(t, err) + require.Equal(t, prepared.Raw, raw, "rotated-key recovery must reuse the exact signed bytes") + } else if failure == "uncertain-broadcast" { + require.Equal(t, sent[0].Hash(), sent[1].Hash(), "stale pending reads must retain the saved transaction") + } + f.mu.Lock() + f.mineLocked(sent[len(sent)-1], true) + f.receiptMissingStatus = true + f.mu.Unlock() before, err := store.Redemptions() require.NoError(t, err) + require.Len(t, before, 1) require.NotEmpty(t, ReconcileSubmitted(t.Context(), store, chain), "missing receipt status must fail reconciliation") after, err := store.Redemptions() require.NoError(t, err) require.Equal(t, before, after, "malformed receipts must leave durable attempts unresolved") - next := *ticket - next.Ticket = &pm.Ticket{PayerAddress: ticket.PayerAddress, Recipient: ticket.Recipient, FaceValue: ticket.FaceValue, WinProb: ticket.WinProb, TicketNonce: 2, RecipientRandHash: ticket.RecipientRandHash, CreationRound: ticket.CreationRound, ParamsExpirationBlock: big.NewInt(10)} - next.Sig = []byte{4, 5, 6} - require.NoError(t, store.StoreWinningTicket(&next)) - validReceipt.Store(true) - snapshotUnavailable.Store(true) - sent := broadcasts.Load() - failures := ProcessRedemptions(t.Context(), store, chain, restartKey, big.NewInt(1)) - require.NotEmpty(t, failures) - after, err = store.Redemptions() - require.NoError(t, err) - require.Equal(t, "confirmed", after[0].Phase, "the worker must reconcile despite an unavailable snapshot") - require.Equal(t, sent, broadcasts.Load()) + next := nextRecoveryTicket(ticket, 4) + next.RecipientRandHash = ticket.RecipientRandHash + next.TicketNonce = 2 + require.NoError(t, store.StoreWinningTicket(next)) + f.mu.Lock() + f.receiptMissingStatus = false + f.snapshotError = true + f.mu.Unlock() + require.NotEmpty(t, ProcessRedemptions(t.Context(), store, chain, restartKey, big.NewInt(1))) + require.Equal(t, "confirmed", ticketPhase(t, store, ticket.Sig), "reconcile despite an unavailable latest snapshot") + require.Len(t, f.transactions(), expectedSends) queued, err := store.PendingPayers() require.NoError(t, err) - require.Equal(t, []ethcommon.Address{ticket.PayerAddress}, queued) - require.Error(t, RetryRedemption(t.Context(), store, contracts, ethcommon.HexToHash(items[0].Hash))) + require.Equal(t, []common.Address{ticket.PayerAddress}, queued) + require.Error(t, RetryRedemption(t.Context(), store, chain.Contracts, common.HexToHash(before[0].Hash))) if failure == "prepared-crash" { floor, err := store.SavedNonceFloor(t.Context(), key.Address()) require.NoError(t, err) @@ -157,22 +640,446 @@ func TestRedemptionRecoveryBeforeAndAfterBroadcast(t *testing.T) { floor, err = store.SavedNonceFloor(t.Context(), restartKey.Address()) require.NoError(t, err) require.Zero(t, floor) - snapshotUnavailable.Store(false) - // A fresh client sees the same stale RPC nonce after restart. - // Saved, confirmed identities must still reserve their nonce. - require.NoError(t, store.Close()) - store, err = OpenRedeemerDB(path) - require.NoError(t, err) - contracts, err = eth.OpenContracts(rpc, ethcommon.HexToAddress("0x1000").Hex()) - require.NoError(t, err) - chain = eth.PaymentChain{Contracts: contracts} - firstRaw = "" + store = reopenRecoveryStore(t, store) + f.mu.Lock() + f.snapshotError = false + f.mu.Unlock() + chain = f.chain(t) require.Empty(t, RedeemPending(t.Context(), store, chain, key, big.NewInt(1), snapshot)) - encoded, err := hex.DecodeString(strings.TrimPrefix(firstRaw, "0x")) + sent = f.transactions() + require.EqualValues(t, 2, sent[len(sent)-1].Nonce(), "confirmed history must advance a fresh client's nonce floor") + } + }) + } +} + +func TestTerminalAttemptRetainsUnresolvedSibling(t *testing.T) { + for _, tc := range []struct { + phase string + minRound int64 + }{{"reverted", 0}, {"invalid", 0}, {"expired", 0}, {"expired", 2}} { + t.Run(fmt.Sprintf("%s/min-round=%d", tc.phase, tc.minRound), func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 11) + f.loseNextResponse = true + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + tickRecovery(t, s, c, key) + sent := f.transactions() + require.Len(t, sent, 2) + f.mine(sent[1], false) + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + require.Equal(t, "reverted", ticketPhase(t, s, ticket.Sig)) + if tc.phase != "reverted" { + require.NoError(t, s.setTicketPhase(t.Context(), ticket.Sig, tc.phase)) + } + // The provider has not accounted for the earlier signed identity. The ticket + // stops retrying, but cannot release its collateral based on one failed hash. + next := nextRecoveryTicket(ticket, 12) + next.CreationRound = max(ticket.CreationRound, tc.minRound) + require.ErrorIs(t, s.storeWinningTickets(t.Context(), ticket.PayerAddress, next.RecipientRandHash, tc.minRound, big.NewInt(1000), []*pm.SignedTicket{next}), pm.ErrPayerUnavailable) + f.mine(sent[0], true) + tickRecovery(t, s, c, key) + require.Equal(t, "confirmed", ticketPhase(t, s, ticket.Sig)) + require.NoError(t, s.storeWinningTickets(t.Context(), ticket.PayerAddress, next.RecipientRandHash, tc.minRound, big.NewInt(1000), []*pm.SignedTicket{next})) + }) + } +} + +func TestOrphanedInclusionRecoversSameNonce(t *testing.T) { + for _, replacements := range []int{0, 1} { + t.Run(fmt.Sprintf("replacements=%d", replacements), func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 13) + f.finalized = false + f.mineAfter = 1 + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, RedemptionOptions{Timeout: 30 * time.Millisecond, MaxReplacements: replacements})) + tickRecovery(t, s, c, key) + require.Eventually(t, func() bool { return ticketPhase(t, s, ticket.Sig) == "mined" }, time.Second, time.Millisecond) + f.mu.Lock() + f.mineAfter = 0 + f.receipts = map[common.Hash]*types.Receipt{} + f.used = map[common.Hash]bool{} + f.mu.Unlock() + tickRecovery(t, s, c, key) + if replacements == 0 { + require.Eventually(t, func() bool { return ticketPhase(t, s, ticket.Sig) == "retryable" }, time.Second, time.Millisecond) + status, err := s.Redemptions() + require.NoError(t, err) + require.Equal(t, "retryable", status[0].Phase) + require.NotEmpty(t, status[0].Error) + require.NoError(t, RetryRedemption(t.Context(), s, c.Contracts, f.transactions()[0].Hash())) + } else { + require.Eventually(t, func() bool { return len(f.transactions()) == 2 }, time.Second, time.Millisecond) + } + sent := f.transactions() + require.Len(t, sent, 2) + require.Equal(t, sent[0].Nonce(), sent[1].Nonce()) + }) + } +} + +func TestRedemptionPersistencePreventsInitialSend(t *testing.T) { + f, s, _, c, key := newRecoveryTest(t, 14) + _, err := s.db.Exec(`CREATE TRIGGER reject_identity BEFORE INSERT ON redemption_transactions BEGIN SELECT RAISE(ABORT,'storage unavailable'); END`) + require.NoError(t, err) + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + require.Empty(t, f.transactions()) + _, err = s.db.Exec("DROP TRIGGER reject_identity") + require.NoError(t, err) + tickRecovery(t, s, c, key) + require.EqualValues(t, 0, f.transactions()[0].Nonce()) +} + +func TestManualRedemptionRetryCannotDisplaceActiveIdentity(t *testing.T) { + f, s, _, c, key := newRecoveryTest(t, 15) + f.loseNextResponse = true + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + first := f.transactions()[0] + require.NoError(t, RetryRedemption(t.Context(), s, c.Contracts, first.Hash())) + require.Equal(t, first.Hash(), f.transactions()[1].Hash()) + // Send another identity only after upstream timeout returns this ticket. + _, err := s.db.Exec("UPDATE redemption_attempts SET phase='retryable'") + require.NoError(t, err) + tickRecovery(t, s, c, key) + require.Len(t, f.transactions(), 3) + require.Error(t, RetryRedemption(t.Context(), s, c.Contracts, first.Hash())) + require.Len(t, f.transactions(), 3) +} + +func TestExpiredAttemptDoesNotReopenRandomness(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 16) + f.rejectNext = true + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + require.NoError(t, s.setTicketPhase(t.Context(), ticket.Sig, "expired")) + next := nextRecoveryTicket(ticket, 17) + next.RecipientRandHash = ticket.RecipientRandHash + next.TicketNonce = 2 + require.ErrorIs(t, s.storeWinningTickets(t.Context(), ticket.PayerAddress, ticket.RecipientRandHash, 0, big.NewInt(10000), []*pm.SignedTicket{next}), ErrInvalidPayment) +} + +func recoveryReplacement(t *testing.T, key *eth.Key, first *types.Transaction) *types.Transaction { + t.Helper() + next, err := key.SignTxForChain(types.NewTx(&types.DynamicFeeTx{ChainID: first.ChainId(), Nonce: first.Nonce(), Gas: first.Gas(), GasFeeCap: big.NewInt(224), GasTipCap: big.NewInt(4), To: first.To(), Data: first.Data(), Value: first.Value()}), first.ChainId()) + require.NoError(t, err) + return next +} + +func TestMinedFailureOutlivesInclusionDeadline(t *testing.T) { + for _, tc := range []struct { + spent, limit int + receiptError, receiptMissing bool + }{{0, 0, false, false}, {1, 1, false, false}, {0, 0, true, false}, {0, 0, false, true}, {1, 1, false, true}, {0, 1, false, false}, {1, 2, false, false}} { + t.Run(fmt.Sprintf("spent=%d/limit=%d/receipt-error=%t/missing=%t", tc.spent, tc.limit, tc.receiptError, tc.receiptMissing), func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 18) + f.finalized = false + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, RedemptionOptions{Timeout: time.Hour, MaxReplacements: tc.limit})) + tickRecovery(t, s, c, key) + s = reopenRecoveryStore(t, s) + last := f.transactions()[0] + if tc.spent > 0 { + replacement := recoveryReplacement(t, key, last) + identity, err := signedIdentity(replacement) require.NoError(t, err) - var signed types.Transaction - require.NoError(t, signed.UnmarshalBinary(encoded)) - require.EqualValues(t, 2, signed.Nonce(), "confirmed history must advance a fresh client's nonce floor") + require.NoError(t, s.recordTransaction(t.Context(), ticket.Sig, identity, last.Hash().Hex())) + require.NoError(t, RetryRedemption(t.Context(), s, c.Contracts, replacement.Hash())) + last = replacement + } + included := last + if tc.receiptMissing && tc.spent > 0 { + included = f.transactions()[0] + } + f.mine(included, false) + _, err := s.db.Exec("UPDATE redemption_transactions SET broadcast_at=?", time.Now().Add(-time.Hour).UTC().Format(time.RFC3339Nano)) + require.NoError(t, err) + _, err = s.db.Exec("UPDATE redemption_attempts SET phase='mined'") + require.NoError(t, err) + _, err = s.db.Exec("UPDATE redemption_transactions SET phase='mined' WHERE hash=?", included.Hash().Hex()) + require.NoError(t, err) + f.mu.Lock() + f.receiptError = tc.receiptError + f.receiptMissing = tc.receiptMissing + f.mu.Unlock() + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, RedemptionOptions{Timeout: time.Nanosecond, MaxReplacements: tc.limit})) + failures := ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1)) + require.Equal(t, tc.receiptError, len(failures) > 0) + require.Eventually(t, func() bool { s.recovery.mu.Lock(); defer s.recovery.mu.Unlock(); return len(s.recovery.tracking) == 0 }, time.Second, time.Millisecond) + require.Equal(t, "mined", ticketPhase(t, s, ticket.Sig), "canonical inclusion must survive an exhausted inclusion timeout") + if included.Hash() != last.Hash() { + var phase string + require.NoError(t, s.db.QueryRow("SELECT phase FROM redemption_transactions WHERE hash=?", last.Hash().Hex()).Scan(&phase)) + require.Equal(t, "submitted", phase, "an included predecessor must not mark the pending replacement mined") + } + ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1)) + require.Len(t, f.transactions(), tc.spent+1, "unfinalized failure must not start another redemption") + }) + } +} + +func TestTerminalTicketKeepsUnresolvedReceipts(t *testing.T) { + for _, phase := range []string{"expired", "invalid"} { + for _, included := range []int{0, 1} { + t.Run(fmt.Sprintf("%s/attempt=%d", phase, included), func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 19) + f.loseNextResponse = true + f.finalized = false + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + tickRecovery(t, s, c, key) + s = reopenRecoveryStore(t, s) + f.mine(f.transactions()[included], true) + require.NoError(t, s.setTicketPhase(t.Context(), ticket.Sig, phase)) + require.Empty(t, ReconcileSubmitted(t.Context(), s, c)) + require.Equal(t, phase, ticketPhase(t, s, ticket.Sig), "unfinalized inclusion must not settle a terminal ticket") + f.mu.Lock() + f.finalized = true + f.mu.Unlock() + require.Empty(t, ReconcileSubmitted(t.Context(), s, c)) + require.Equal(t, "confirmed", ticketPhase(t, s, ticket.Sig)) + }) + } + } +} + +func TestRevertedReplacementReleasesSameNonceLiability(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 20) + tickRecovery(t, s, c, key) + s = reopenRecoveryStore(t, s) + first := f.transactions()[0] + replacement := recoveryReplacement(t, key, first) + identity, err := signedIdentity(replacement) + require.NoError(t, err) + require.NoError(t, s.recordTransaction(t.Context(), ticket.Sig, identity, first.Hash().Hex())) + require.NoError(t, RetryRedemption(t.Context(), s, c.Contracts, replacement.Hash())) + f.mine(replacement, false) + require.NotEmpty(t, ReconcileSubmitted(t.Context(), s, c)) + require.Equal(t, "reverted", ticketPhase(t, s, ticket.Sig)) + next := nextRecoveryTicket(ticket, 21) + require.NoError(t, s.storeWinningTickets(t.Context(), ticket.PayerAddress, next.RecipientRandHash, 0, big.NewInt(1000), []*pm.SignedTicket{next}), "a finalized replacement consumes every sibling with the same sender and nonce") +} + +func TestManualRetryFailureKeepsActiveMonitor(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 101) + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, RedemptionOptions{Timeout: time.Hour, MaxReplacements: 1})) + tickRecovery(t, s, c, key) + first := f.transactions()[0] + f.mu.Lock() + f.rejectNext = true + f.mu.Unlock() + require.Error(t, RetryRedemption(t.Context(), s, c.Contracts, first.Hash())) + require.Equal(t, "submitted", ticketPhase(t, s, ticket.Sig)) + tickRecovery(t, s, c, key) + sent := f.transactions() + s.recovery.mu.Lock() + roots := len(s.recovery.tracking) + s.recovery.mu.Unlock() + require.Equal(t, 1, roots) + require.Len(t, sent, 2, "an identical-byte manual retry must not admit a fresh nonce while the original monitor is active") +} + +func TestConsumedNonceResolvesOtherTicketHistory(t *testing.T) { + f := newRecoveryRPC(t) + s, a := f.store(t, 103) + ca, cb, key := f.chain(t), f.chain(t), recoveryKey(t) + b := nextRecoveryTicket(a, 104) + b.PayerAddress = common.HexToAddress("0x9999") + require.NoError(t, s.StoreWinningTicket(b)) + a0, err := ca.PrepareRedemption(t.Context(), key, big.NewInt(1), redeemTicket(a)) + require.NoError(t, err) + require.NoError(t, s.recordPrepared(t.Context(), a, a0)) + f.mu.Lock() + f.rejectNext = true + f.mu.Unlock() + require.Error(t, broadcastRedemption(t.Context(), s, ca.Contracts, a.Sig, a0)) + b0, err := cb.PrepareRedemption(t.Context(), key, big.NewInt(1), redeemTicket(b)) + require.NoError(t, err) + require.Equal(t, a0.Nonce, b0.Nonce) + require.NoError(t, s.recordPrepared(t.Context(), b, b0)) + require.NoError(t, broadcastRedemption(t.Context(), s, cb.Contracts, b.Sig, b0)) + a1, err := ca.PrepareRedemption(t.Context(), key, big.NewInt(1), redeemTicket(a)) + require.NoError(t, err) + require.EqualValues(t, 1, a1.Nonce) + require.NoError(t, s.recordPrepared(t.Context(), a, a1)) + require.NoError(t, broadcastRedemption(t.Context(), s, ca.Contracts, a.Sig, a1)) + sent := f.transactions() + f.mine(sent[1], true) + f.mine(sent[2], false) + require.NotEmpty(t, ReconcileSubmitted(t.Context(), s, ca)) + require.Equal(t, "confirmed", ticketPhase(t, s, b.Sig)) + require.Equal(t, "reverted", ticketPhase(t, s, a.Sig)) + next := nextRecoveryTicket(a, 105) + require.NoError(t, s.storeWinningTickets(t.Context(), a.PayerAddress, next.RecipientRandHash, 0, big.NewInt(1000), []*pm.SignedTicket{next}), "B consumed nonce zero and A's nonce one failed; A has no remaining liability") +} + +func TestMultipleFinalizedFailuresResolveEveryGroup(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 106) + var originals []*types.Transaction + for i := 0; i < 2; i++ { + prepared, err := c.PrepareRedemption(t.Context(), key, big.NewInt(1), redeemTicket(ticket)) + require.NoError(t, err) + require.NoError(t, s.recordPrepared(t.Context(), ticket, prepared)) + require.NoError(t, broadcastRedemption(t.Context(), s, c.Contracts, ticket.Sig, prepared)) + first := f.transactions()[2*i] + originals = append(originals, first) + replacement := recoveryReplacement(t, key, first) + identity, err := signedIdentity(replacement) + require.NoError(t, err) + require.NoError(t, s.recordTransaction(t.Context(), ticket.Sig, identity, first.Hash().Hex())) + require.NoError(t, broadcastRedemption(t.Context(), s, c.Contracts, ticket.Sig, identity)) + if i == 0 { + _, err = s.db.Exec("UPDATE redemption_attempts SET phase='retryable' WHERE sig=?", ticket.Sig) + require.NoError(t, err) + } + } + for _, tx := range originals { + f.mine(tx, false) + } + require.NotEmpty(t, ReconcileSubmitted(t.Context(), s, c)) + history, err := s.Redemptions() + require.NoError(t, err) + require.Len(t, history, 4) + next := nextRecoveryTicket(ticket, 107) + require.NoError(t, s.storeWinningTickets(t.Context(), ticket.PayerAddress, next.RecipientRandHash, 0, big.NewInt(1000), []*pm.SignedTicket{next}), "both finalized failed nonces must resolve their replacement siblings") +} + +func TestLiveAcknowledgementRestoresNonce(t *testing.T) { + for _, mode := range []string{"manual", "manual-external-handle", "finalized"} { + t.Run(mode, func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 111) + f.loseNextResponse = true + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, RedemptionOptions{Timeout: time.Hour, MaxReplacements: 1})) + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + first := f.transactions()[0] + if mode != "finalized" { + retryStore := s + if mode == "manual-external-handle" { + var seq int + var name, path string + require.NoError(t, s.db.QueryRow("PRAGMA database_list").Scan(&seq, &name, &path)) + var err error + retryStore, err = OpenRedeemerDB(path) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, retryStore.Close()) }) + } + require.NoError(t, RetryRedemption(t.Context(), retryStore, c.Contracts, first.Hash())) + } else { + f.mine(first, true) + require.Empty(t, ReconcileSubmitted(t.Context(), s, c)) + } + floor, err := s.SavedNonceFloor(t.Context(), key.Address()) + require.NoError(t, err) + require.EqualValues(t, 1, floor) + next := nextRecoveryTicket(ticket, 112) + next.PayerAddress = common.HexToAddress("0x9999") + require.NoError(t, s.StoreWinningTicket(next)) + f.mu.Lock() + f.staleNonce = true + f.mu.Unlock() + tickRecovery(t, s, c, key) + sent := f.transactions() + require.EqualValues(t, 1, sent[len(sent)-1].Nonce(), "new submission must honor a durable acknowledgement received since this worker started") + }) + } +} + +func TestUsedTransitionRecoversAfterStorageFailure(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 222) + f.loseNextResponse = true + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + s = reopenRecoveryStore(t, s) + f.mu.Lock() + f.used[ticket.Hash()] = true + f.mu.Unlock() + _, err := s.db.Exec(`CREATE TRIGGER fail_settlement BEFORE UPDATE OF redeemed_at ON winning_tickets BEGIN SELECT RAISE(ABORT,'injected settlement failure'); END`) + require.NoError(t, err) + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + require.Equal(t, "retryable", ticketPhase(t, s, ticket.Sig), "failed settlement must roll back the terminal summary") + _, err = s.db.Exec("DROP TRIGGER fail_settlement") + require.NoError(t, err) + s = reopenRecoveryStore(t, s) + tickRecovery(t, s, c, key) + var redeemed bool + require.NoError(t, s.db.QueryRow("SELECT redeemed_at IS NOT NULL FROM winning_tickets WHERE sig=?", ticket.Sig).Scan(&redeemed)) + if !redeemed { + t.Errorf("summary=%s but redeemed_at remains NULL even after restart and retry", ticketPhase(t, s, ticket.Sig)) + } + next := nextRecoveryTicket(ticket, 223) + require.NoError(t, s.storeWinningTickets(t.Context(), ticket.PayerAddress, next.RecipientRandHash, 0, big.NewInt(1000), []*pm.SignedTicket{next}), "a finalized-used ticket must not strand collateral") +} + +func TestRejectedReplacementKeepsPredecessorReconcilable(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 225) + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, RedemptionOptions{Timeout: time.Hour, MaxReplacements: 1})) + tickRecovery(t, s, c, key) + original := f.transactions()[0] + s = reopenRecoveryStore(t, s) + replacement := recoveryReplacement(t, key, original) + identity, err := signedIdentity(replacement) + require.NoError(t, err) + require.NoError(t, s.recordTransaction(t.Context(), ticket.Sig, identity, original.Hash().Hex())) + f.mu.Lock() + f.rejectNext = true + f.mu.Unlock() + require.NoError(t, StartRedemptionRecovery(t.Context(), s, c, key, RedemptionOptions{Timeout: time.Hour, MaxReplacements: 1})) + require.NotEmpty(t, s.recovery.resume(t.Context())) + require.Equal(t, "retryable", ticketPhase(t, s, ticket.Sig)) + var phase string + require.NoError(t, s.db.QueryRow("SELECT phase FROM redemption_transactions WHERE hash=?", original.Hash().Hex()).Scan(&phase)) + require.Equal(t, "submitted", phase) + f.mine(original, true) + require.Empty(t, ReconcileSubmitted(t.Context(), s, c)) + require.Equal(t, "confirmed", ticketPhase(t, s, ticket.Sig)) +} + +func TestFinalizedUsePollingRetainsOnlyUnresolvedWork(t *testing.T) { + for _, tc := range []struct { + name string + unresolved, mined bool + }{{"resolved revert", false, false}, {"unresolved revert", true, false}, {"mined", false, true}} { + unresolved := tc.unresolved + t.Run(tc.name, func(t *testing.T) { + f, s, ticket, c, key := newRecoveryTest(t, 226) + f.loseNextResponse = unresolved + if tc.mined { + f.mineAfter = 1 + f.finalized = false + } + if unresolved { + require.NotEmpty(t, ProcessRedemptions(t.Context(), s, c, key, big.NewInt(1))) + } + tickRecovery(t, s, c, key) + if tc.mined { + require.Eventually(t, func() bool { return ticketPhase(t, s, ticket.Sig) == "mined" }, time.Second, time.Millisecond) + } + s = reopenRecoveryStore(t, s) + if tc.mined { + f.mu.Lock() + f.finalized = true + f.receiptMissing = true + f.mu.Unlock() + } else { + sent := f.transactions() + f.mine(sent[len(sent)-1], false) + require.NotEmpty(t, ReconcileSubmitted(t.Context(), s, c)) + require.Equal(t, "reverted", ticketPhase(t, s, ticket.Sig)) + } + r := &redemptionRecovery{store: s, chain: c} + f.mu.Lock() + before := f.contractCalls + f.mu.Unlock() + require.Empty(t, r.reconcileUsed(t.Context())) + require.Empty(t, r.reconcileUsed(t.Context())) + f.mu.Lock() + calls := f.contractCalls - before + f.mu.Unlock() + require.Equal(t, unresolved || tc.mined, calls > 0, "only unresolved or mined tickets need contract-state polling") + if unresolved || tc.mined { + f.mu.Lock() + f.used[ticket.Hash()] = true + f.receiptMissing = true + f.mu.Unlock() + require.Empty(t, ReconcileSubmitted(t.Context(), s, c)) + require.Empty(t, r.reconcileUsed(t.Context())) + require.Equal(t, "used", ticketPhase(t, s, ticket.Sig)) + var redeemed bool + require.NoError(t, s.db.QueryRow("SELECT redeemed_at IS NOT NULL FROM winning_tickets WHERE sig=?", ticket.Sig).Scan(&redeemed)) + require.True(t, redeemed) } }) } diff --git a/orchestrator/redemption_regression_test.go b/orchestrator/redemption_regression_test.go deleted file mode 100644 index 44c6c8c..0000000 --- a/orchestrator/redemption_regression_test.go +++ /dev/null @@ -1,112 +0,0 @@ -package orchestrator - -import ( - "encoding/hex" - "encoding/json" - "fmt" - "math/big" - "net/http" - "net/http/httptest" - "path/filepath" - "strings" - "sync/atomic" - "testing" - - ethcommon "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/livepeer/node/eth" - "github.com/livepeer/node/internal/test" - "github.com/livepeer/node/pm" - "github.com/stretchr/testify/require" -) - -func TestRedemptionEligibility(t *testing.T) { - for _, tc := range []struct { - block, round int64 - active bool - lookupError bool - }{{9, 5, true, false}, {10, 5, true, false}, {11, 5, true, false}, {10, 5, false, false}, {10, 5, true, true}, {10, 8, true, false}} { - t.Run(fmt.Sprintf("block=%d/round=%d/active=%t/error=%t", tc.block, tc.round, tc.active, tc.lookupError), func(t *testing.T) { - path, passwordPath := test.WriteFixedKeystore(t) - key, err := eth.OpenKeystoreFile(path, passwordPath) - require.NoError(t, err) - store, err := OpenRedeemerDB(filepath.Join(t.TempDir(), "payment.sqlite")) - require.NoError(t, err) - defer store.Close() - ticket := &pm.SignedTicket{Ticket: &pm.Ticket{PayerAddress: ethcommon.HexToAddress("0x1234"), Recipient: ethcommon.HexToAddress("0x5678"), FaceValue: big.NewInt(10), WinProb: big.NewInt(100), TicketNonce: 1, RecipientRandHash: ethcommon.HexToHash("0xabcd"), CreationRound: 5, ParamsExpirationBlock: big.NewInt(10)}, Sig: []byte{1, 2, 3}, RecipientRand: big.NewInt(4)} - require.NoError(t, store.StoreWinningTicket(ticket)) - var broadcasts atomic.Int32 - rpcServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - var req struct { - Method string - Params []json.RawMessage - } - require.NoError(t, json.NewDecoder(r.Body).Decode(&req)) - var result any = "0x" - switch req.Method { - case "eth_call": - result = "0x" + strings.Repeat("0", 60) + "2000" - var call struct{ Input string } - require.NoError(t, json.Unmarshal(req.Params[0], &call)) - selector := func(sig string) bool { - return strings.HasPrefix(call.Input, fmt.Sprintf("0x%x", crypto.Keccak256([]byte(sig))[:4])) - } - switch { - case selector("lastInitializedRound()"): - result = fmt.Sprintf("0x%064x", tc.round) - case selector("isActiveTranscoder(address)"): - require.True(t, strings.HasSuffix(call.Input, fmt.Sprintf("%064x", ticket.Recipient.Bytes()))) - if tc.lookupError { - require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "error": map[string]any{"code": -32000, "message": "unavailable"}})) - return - } - result = fmt.Sprintf("0x%064x", 0) - if tc.active { - result = fmt.Sprintf("0x%064x", 1) - } - } - case "eth_getBlockByNumber": - result = &types.Header{Number: big.NewInt(tc.block), Difficulty: new(big.Int), BaseFee: big.NewInt(1)} - case "eth_estimateGas": - result = "0x5208" - case "eth_maxPriorityFeePerGas": - result = "0x1" - case "eth_getTransactionCount": - result = "0x1" - case "eth_sendRawTransaction": - broadcasts.Add(1) - var encoded string - require.NoError(t, json.Unmarshal(req.Params[0], &encoded)) - raw, err := hex.DecodeString(strings.TrimPrefix(encoded, "0x")) - require.NoError(t, err) - var tx types.Transaction - require.NoError(t, tx.UnmarshalBinary(raw)) - result = tx.Hash().Hex() - // The exact transaction identity must exist before sending. - items, err := store.Redemptions() - require.NoError(t, err) - require.Equal(t, []RedemptionStatus{{Hash: result.(string), Phase: "broadcast"}}, items) - case "eth_getTransactionReceipt": - result = nil - default: - t.Errorf("unexpected RPC method %s", req.Method) - } - require.NoError(t, json.NewEncoder(w).Encode(map[string]any{"jsonrpc": "2.0", "id": 1, "result": result})) - })) - defer rpcServer.Close() - rpc, err := eth.OpenRPC(rpcServer.URL) - require.NoError(t, err) - contracts, err := eth.OpenContracts(rpc, "0x0000000000000000000000000000000000001000") - require.NoError(t, err) - failures := ProcessRedemptions(t.Context(), store, eth.PaymentChain{Contracts: contracts}, key, big.NewInt(1)) - require.Equal(t, tc.lookupError, len(failures) != 0, "%v", failures) - expired := tc.round > 7 - redeemed := tc.block >= 10 && !expired && tc.active && !tc.lookupError - require.Equal(t, redeemed, broadcasts.Load() == 1) - queued, err := store.PendingPayers() - require.NoError(t, err) - require.Equal(t, !redeemed && !expired, len(queued) == 1) - }) - } -} diff --git a/orchestrator/redemption_store.go b/orchestrator/redemption_store.go index 5d2cfc7..bc0a5d8 100644 --- a/orchestrator/redemption_store.go +++ b/orchestrator/redemption_store.go @@ -10,54 +10,77 @@ import ( "time" ethcommon "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/livepeer/node/eth" "github.com/livepeer/node/pm" ) -// SavedNonceFloor reserves every signed identity across process restarts, -// including confirmed/reverted attempts that an RPC may omit from pending. +// SavedNonceFloor restores only acknowledged sends, following upstream backend's +// Update-after-send rule. A rejected/prepared identity must not create a gap. func (s *RedeemerDB) SavedNonceFloor(ctx context.Context, address ethcommon.Address) (uint64, error) { var raw string - err := s.db.QueryRowContext(ctx, "SELECT nonce FROM redemption_attempts WHERE redeemer_address=? AND nonce IS NOT NULL ORDER BY length(nonce) DESC, nonce DESC LIMIT 1", address.Hex()).Scan(&raw) + err := s.db.QueryRowContext(ctx, `SELECT nonce FROM redemption_transactions WHERE redeemer_address=? AND acknowledged=1 ORDER BY length(nonce) DESC,nonce DESC LIMIT 1`, address.Hex()).Scan(&raw) if errors.Is(err, sql.ErrNoRows) { return 0, nil } if err != nil { return 0, err } - nonce, err := strconv.ParseUint(raw, 10, 64) - if err != nil || nonce == math.MaxUint64 { + n, err := strconv.ParseUint(raw, 10, 64) + if err != nil || n == math.MaxUint64 { return 0, errors.New("invalid or exhausted saved redemption nonce") } - return nonce + 1, nil + return n + 1, nil } func (s *RedeemerDB) recordPrepared(ctx context.Context, ticket *pm.SignedTicket, prepared eth.SignedTransaction) error { + return s.recordTransaction(ctx, ticket.Sig, prepared, "") +} + +func (s *RedeemerDB) recordTransaction(ctx context.Context, sig []byte, prepared eth.SignedTransaction, previous string) error { tx, err := s.db.BeginTx(ctx, nil) if err != nil { return err } defer tx.Rollback() - var blocked int - if err := tx.QueryRow("SELECT COUNT(*) FROM redemption_attempts WHERE phase IN ('prepared','broadcast')").Scan(&blocked); err != nil { + root := prepared.Hash.Hex() + if previous != "" { + if err := tx.QueryRowContext(ctx, "SELECT root_hash FROM redemption_transactions WHERE hash=? AND sig=?", previous, sig).Scan(&root); err != nil { + return err + } + } + now := time.Now().UTC().Format(time.RFC3339Nano) + var result sql.Result + if previous == "" { + result, err = tx.ExecContext(ctx, `INSERT INTO redemption_attempts(sig,attempted_at,phase,raw_transaction,redeemer_address,nonce) + VALUES(?,?,'prepared',?,?,?) ON CONFLICT(sig) DO UPDATE SET attempted_at=excluded.attempted_at,phase='prepared',error=NULL,raw_transaction=excluded.raw_transaction,redeemer_address=excluded.redeemer_address,nonce=excluded.nonce WHERE redemption_attempts.phase='retryable'`, sig, now, prepared.Raw, prepared.From.Hex(), strconv.FormatUint(prepared.Nonce, 10)) + } else { + result, err = tx.ExecContext(ctx, `UPDATE redemption_attempts SET attempted_at=?,phase='prepared',error=NULL,raw_transaction=?,redeemer_address=?,nonce=? WHERE sig=? AND phase IN ('prepared','broadcast','submitted','mined') AND EXISTS(SELECT 1 FROM winning_tickets WHERE sig=? AND tx_hash=?)`, now, prepared.Raw, prepared.From.Hex(), strconv.FormatUint(prepared.Nonce, 10), sig, sig, previous) + } + if err != nil { return err } - if blocked != 0 { - return errors.New("unresolved redemption must be reconciled before preparing another") + if n, e := result.RowsAffected(); e != nil || n != 1 { + return errors.New("redemption is already active or resolved") } - _, err = tx.Exec("INSERT INTO redemption_attempts(sig,attempted_at,phase,raw_transaction,redeemer_address,nonce) VALUES(?,?,'prepared',?,?,?)", ticket.Sig, time.Now().UTC().Format(time.RFC3339Nano), prepared.Raw, prepared.From.Hex(), strconv.FormatUint(prepared.Nonce, 10)) + _, err = tx.ExecContext(ctx, `INSERT INTO redemption_transactions(hash,sig,root_hash,replaces_hash,raw_transaction,redeemer_address,nonce,phase,prepared_at) VALUES(?,?,?,?,?,?,?,'prepared',?) ON CONFLICT(hash) DO UPDATE SET phase='prepared',error=NULL`, prepared.Hash.Hex(), sig, root, nullString(previous), prepared.Raw, prepared.From.Hex(), strconv.FormatUint(prepared.Nonce, 10), now) if err != nil { return err } - _, err = tx.Exec("UPDATE winning_tickets SET tx_hash=? WHERE sig=?", prepared.Hash.Hex(), ticket.Sig) + _, err = tx.ExecContext(ctx, "UPDATE winning_tickets SET tx_hash=? WHERE sig=?", prepared.Hash.Hex(), sig) if err != nil { return err } return tx.Commit() } +func nullString(s string) any { + if s == "" { + return nil + } + return s +} -// RedemptionStatus is safe to display: it excludes recipient randomness, -// signatures, and signed transaction bytes. +// RedemptionStatus excludes recipient randomness, signatures and signed bytes. type RedemptionStatus struct { Hash string `json:"hash,omitempty"` Phase string `json:"phase"` @@ -65,49 +88,173 @@ type RedemptionStatus struct { } func (s *RedeemerDB) Redemptions() ([]RedemptionStatus, error) { - rows, err := s.db.Query("SELECT COALESCE(w.tx_hash,''),a.phase,COALESCE(a.error,'') FROM redemption_attempts a JOIN winning_tickets w ON w.sig=a.sig ORDER BY a.attempted_at") + rows, err := s.db.Query(`SELECT hash,phase,error FROM ( + SELECT hash,CASE WHEN phase='sending' THEN 'broadcast' ELSE phase END AS phase,COALESCE(error,'') AS error,rowid AS sequence FROM redemption_transactions + UNION ALL SELECT '',phase,COALESCE(error,''),9223372036854775807 FROM redemption_attempts a WHERE a.phase IN ('used','awaiting_finality') OR NOT EXISTS(SELECT 1 FROM redemption_transactions t WHERE t.sig=a.sig)) ORDER BY sequence`) if err != nil { return nil, err } defer rows.Close() - result := []RedemptionStatus{} + out := []RedemptionStatus{} for rows.Next() { var item RedemptionStatus if err := rows.Scan(&item.Hash, &item.Phase, &item.Error); err != nil { return nil, err } - result = append(result, item) + out = append(out, item) } - return result, rows.Err() + return out, rows.Err() } -// RetryRedemption is an explicit operator action after an uncertain broadcast. -// It can only send the identical stored transaction, preserving its nonce/hash. +// RetryRedemption preserves the existing explicit identical-byte retry command. +// Historical identities are inspectable, but cannot replace a newer active job. func RetryRedemption(ctx context.Context, store *RedeemerDB, contracts *eth.Contracts, hash ethcommon.Hash) error { var sig, raw []byte - var phase string - err := store.db.QueryRow("SELECT a.sig,a.raw_transaction,a.phase FROM redemption_attempts a JOIN winning_tickets w ON w.sig=a.sig WHERE w.tx_hash=?", hash.Hex()).Scan(&sig, &raw, &phase) + err := store.db.QueryRowContext(ctx, `SELECT t.sig,t.raw_transaction FROM redemption_transactions t JOIN redemption_attempts a ON a.sig=t.sig JOIN winning_tickets w ON w.sig=t.sig WHERE t.hash=? AND w.tx_hash=t.hash AND a.phase IN ('prepared','broadcast','submitted','retryable') AND t.phase IN ('prepared','broadcast','submitted','retryable')`, hash.Hex()).Scan(&sig, &raw) + if err != nil { + return fmt.Errorf("redemption has no retryable signed transaction: %w", err) + } + return broadcastRedemption(ctx, store, contracts, sig, eth.SignedTransaction{Hash: hash, Raw: raw}) +} + +func broadcastRedemption(ctx context.Context, store *RedeemerDB, contracts *eth.Contracts, sig []byte, identity eth.SignedTransaction) error { + return broadcastSavedRedemption(ctx, store, contracts, sig, identity, false) +} + +// Gap repair may replay older signed bytes without displacing current work or +// reopening a terminal ticket. Manual retries retain their current-only guard. +func broadcastSavedRedemption(ctx context.Context, store *RedeemerDB, contracts *eth.Contracts, sig []byte, identity eth.SignedTransaction, historical bool) error { + tx, err := store.db.BeginTx(ctx, nil) if err != nil { return err } - if len(raw) == 0 || (phase != "prepared" && phase != "broadcast" && phase != "submitted") { - return errors.New("redemption has no retryable signed transaction") + defer tx.Rollback() + // Claim the send atomically, including against another manual retry. A crash + // in 'sending' is recovered as an uncertain send using the same stored bytes. + var previousPhase string + if err := tx.QueryRowContext(ctx, "SELECT phase FROM redemption_transactions WHERE hash=? AND sig=?", identity.Hash.Hex(), sig).Scan(&previousPhase); err != nil { + return err } - return broadcastRedemption(ctx, store, contracts, sig, eth.SignedTransaction{Hash: hash, Raw: raw}) + // Closed histories cannot be manually retried. Keep their recorded outcome + // durable during replay as well: a crash must not replace it with 'sending'. + preserve := historical && (terminalRedemption(previousPhase) || previousPhase == "superseded") + claimedPhase := "sending" + if preserve { + claimedPhase = previousPhase + } + result, err := tx.ExecContext(ctx, `UPDATE redemption_transactions SET phase=?,error=NULL,broadcast_at=CASE WHEN NOT ? AND phase IN ('prepared','retryable') THEN ? ELSE COALESCE(broadcast_at,?) END WHERE hash=? AND sig=? AND (phase IN ('prepared','broadcast','submitted','retryable') OR (? AND phase IN ('sending','mined','used','expired','invalid','superseded'))) AND EXISTS(SELECT 1 FROM redemption_attempts a JOIN winning_tickets w ON w.sig=a.sig WHERE a.sig=? AND (? OR (w.tx_hash=? AND a.phase IN ('prepared','broadcast','submitted','retryable'))))`, claimedPhase, historical, time.Now().UTC().Format(time.RFC3339Nano), time.Now().UTC().Format(time.RFC3339Nano), identity.Hash.Hex(), sig, historical, sig, historical, identity.Hash.Hex()) + if err != nil { + return err + } + if n, e := result.RowsAffected(); e != nil || n != 1 { + return errors.New("redemption is already sending, mined or resolved") + } + if _, err = tx.ExecContext(ctx, `UPDATE redemption_attempts SET phase='broadcast',error=NULL WHERE sig=? AND phase IN ('prepared','broadcast','submitted','retryable','mined') AND EXISTS(SELECT 1 FROM winning_tickets WHERE sig=? AND tx_hash=?)`, sig, sig, identity.Hash.Hex()); err != nil { + return err + } + if err = tx.Commit(); err != nil { + return err + } + sendErr := contracts.Broadcast(ctx, identity) + phase := "submitted" + var message any + ack := 0 + if sendErr != nil { + phase = "retryable" + // Resending acknowledged bytes does not undo their live monitor or + // acknowledgment, even if this particular resend is rejected. + if previousPhase == "submitted" { + phase = "submitted" + } + message = sendErr.Error() + } else { + ack = 1 + } + if preserve { + phase = previousPhase + } + tx, err = store.db.BeginTx(context.WithoutCancel(ctx), nil) + if err != nil { + return errors.Join(sendErr, err) + } + defer tx.Rollback() + _, err = tx.Exec(`UPDATE redemption_transactions SET phase=?,error=?,acknowledged=max(acknowledged,?) WHERE hash=? AND phase=?`, phase, message, ack, identity.Hash.Hex(), claimedPhase) + if err == nil { + _, err = tx.Exec(`UPDATE redemption_attempts SET phase=?,error=? WHERE sig=? AND phase='broadcast' AND EXISTS(SELECT 1 FROM winning_tickets WHERE sig=? AND tx_hash=?)`, phase, message, sig, sig, identity.Hash.Hex()) + } + if err != nil { + return errors.Join(sendErr, err) + } + return errors.Join(sendErr, tx.Commit()) } -func broadcastRedemption(ctx context.Context, store *RedeemerDB, contracts *eth.Contracts, sig []byte, tx eth.SignedTransaction) error { - result, err := store.db.Exec("UPDATE redemption_attempts SET phase='broadcast',error=NULL WHERE sig=? AND phase IN ('prepared','broadcast','submitted')", sig) +func signedIdentity(tx *types.Transaction) (eth.SignedTransaction, error) { + raw, err := tx.MarshalBinary() + if err != nil { + return eth.SignedTransaction{}, err + } + from, err := types.Sender(types.LatestSignerForChainID(tx.ChainId()), tx) + if err != nil { + return eth.SignedTransaction{}, err + } + return eth.SignedTransaction{Hash: tx.Hash(), Raw: raw, Nonce: tx.Nonce(), From: from}, nil +} + +// finishTicket never discards transaction history or the epoch exposure marker. +func (s *RedeemerDB) finishTicket(ctx context.Context, sig []byte, phase string, hash string) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + defer tx.Rollback() + if hash == "" && phase != "used" { + var latest sql.NullString + if err := tx.QueryRowContext(ctx, "SELECT tx_hash FROM winning_tickets WHERE sig=?", sig).Scan(&latest); err != nil { + return err + } + hash = latest.String + } + if phase == "confirmed" || phase == "reverted" { + // Every finalized receipt consumes the sender's nonce across tickets and + // retry groups. Other nonces remain uncertain. Retire identities before + // applying the ticket outcome, which may already be a finalized success. + _, err = tx.ExecContext(ctx, `UPDATE redemption_transactions SET phase='superseded',error=NULL WHERE hash!=? AND (redeemer_address,nonce)=(SELECT redeemer_address,nonce FROM redemption_transactions WHERE hash=?) AND phase NOT IN ('confirmed','reverted')`, hash, hash) + if err == nil { + _, err = tx.ExecContext(ctx, `UPDATE redemption_attempts SET phase='retryable',error=NULL WHERE sig!=? AND phase IN ('prepared','broadcast','submitted','mined') AND EXISTS(SELECT 1 FROM winning_tickets w JOIN redemption_transactions t ON t.hash=w.tx_hash WHERE w.sig=redemption_attempts.sig AND t.phase='superseded' AND (t.redeemer_address,t.nonce)=(SELECT redeemer_address,nonce FROM redemption_transactions WHERE hash=?))`, sig, hash) + } + var message any + if phase == "reverted" { + message = "transaction reverted" + } + if err == nil { + _, err = tx.ExecContext(ctx, `UPDATE redemption_transactions SET phase=?,error=?,acknowledged=1 WHERE hash=?`, phase, message, hash) + } + if err != nil { + return err + } + } + // Creating the summary and settling a terminal outcome must commit together. + result, err := tx.ExecContext(ctx, `INSERT INTO redemption_attempts(sig,attempted_at,phase) VALUES(?,?,?) ON CONFLICT(sig) DO UPDATE SET phase=excluded.phase,error=NULL WHERE redemption_attempts.phase NOT IN ('confirmed','used') OR excluded.phase='confirmed' OR (excluded.phase='used' AND redemption_attempts.phase!='confirmed')`, sig, time.Now().UTC().Format(time.RFC3339Nano), phase) if err != nil { return err } - if n, err := result.RowsAffected(); err != nil || n != 1 { - return errors.New("redemption is already resolved") + if n, err := result.RowsAffected(); err != nil { + return err + } else if n == 0 { + return tx.Commit() + } + if phase == "confirmed" || phase == "used" { + _, err = tx.ExecContext(ctx, `UPDATE winning_tickets SET redeemed_at=COALESCE(redeemed_at,?),tx_hash=COALESCE(?,tx_hash) WHERE sig=?`, time.Now().UTC().Format(time.RFC3339Nano), nullString(hash), sig) } - if err := contracts.Broadcast(ctx, tx); err != nil { - _, dbErr := store.db.Exec("UPDATE redemption_attempts SET error=? WHERE sig=? AND phase='broadcast'", err.Error(), sig) - return errors.Join(fmt.Errorf("transaction %s broadcast outcome uncertain: %w", tx.Hash.Hex(), err), dbErr) + if err == nil && (phase == "expired" || phase == "invalid") { + // Submission eligibility can end before an uncertain send finalizes. + // Keep every predecessor receipt-visible until actual settlement. + _, err = tx.ExecContext(ctx, `UPDATE redemption_transactions SET phase=?,error=NULL WHERE hash=? AND phase NOT IN ('confirmed','reverted','superseded')`, phase, hash) + } else if err == nil && phase != "reverted" { + _, err = tx.ExecContext(ctx, `UPDATE redemption_transactions SET phase=CASE WHEN hash=? THEN ? ELSE 'superseded' END,error=NULL,acknowledged=CASE WHEN hash=? AND ?='confirmed' THEN 1 ELSE acknowledged END WHERE sig=? AND phase NOT IN ('confirmed','reverted')`, hash, phase, hash, phase, sig) } - _, err = store.db.Exec("UPDATE redemption_attempts SET phase='submitted',error=NULL WHERE sig=? AND phase='broadcast'", sig) - return err + if err != nil { + return err + } + return tx.Commit() } From d41607f63ae6f26f03711014862f1b9f2db780b8 Mon Sep 17 00:00:00 2001 From: Josh Allmann Date: Mon, 26 Oct 2026 21:14:37 -0700 Subject: [PATCH 2/2] test: fix staticcheck findings --- eth/transactionmanager_test.go | 1 + orchestrator/redemption_recovery_test.go | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/eth/transactionmanager_test.go b/eth/transactionmanager_test.go index 9f9f8b8..3437886 100644 --- a/eth/transactionmanager_test.go +++ b/eth/transactionmanager_test.go @@ -191,6 +191,7 @@ func TestTransactionManager_Replace(t *testing.T) { expErr = errors.New("SendTx error") eth.err["SendTransaction"] = expErr tx, err = tm.replace(stubTx) + assert.NotNil(tx) assert.EqualError(err, expErr.Error()) eth.err["SendTransaction"] = nil diff --git a/orchestrator/redemption_recovery_test.go b/orchestrator/redemption_recovery_test.go index 18baad4..f77cd4a 100644 --- a/orchestrator/redemption_recovery_test.go +++ b/orchestrator/redemption_recovery_test.go @@ -193,7 +193,7 @@ func (f *recoveryRPC) serve(w http.ResponseWriter, r *http.Request) { if f.loseNextResponse { f.loseNextResponse = false f.mu.Unlock() - http.Error(w, "lost response", 503) + http.Error(w, "lost response", http.StatusServiceUnavailable) return } case "eth_getTransactionReceipt":