From 24fb52e5180152f9fdfdd23fee62bcf299b29ada Mon Sep 17 00:00:00 2001 From: Nishk Patel Date: Wed, 21 Oct 2026 21:00:05 +0000 Subject: [PATCH] dev: add [development.force_mismatch] to force a bank hash mismatch Adds a firedancer-dev development knob to deliberately force a bank hash mismatch, for exercising the mismatch-handling paths. [development.force_mismatch] enabled = false at_slot = 0 # force on a specific slot after = 0 # force after this many slots; both unset => first slot One config, two consumers, each with its own reaction: - backtest: the backt tile forces its ledger comparison to mismatch, which logs the mismatch and exits (as a real mismatch does). - live/network: the replay tile forces the Alpenglow footer comparison to mismatch once (latched), marking the block dead and continuing. In a backtest run both tiles exist, so the backt tile owns the knob and replay's copy is disarmed in the backtest command topology. at_slot and after are mutually exclusive and rejected together at config-parse time. --- src/app/firedancer-dev/commands/backtest.c | 6 ++++++ src/app/firedancer/config/default.toml | 19 +++++++++++++++++++ src/app/firedancer/topology.c | 7 +++++++ src/app/shared/fd_config.h | 6 ++++++ src/app/shared/fd_config_json.c | 7 ++++++- src/app/shared/fd_config_parse.c | 9 ++++++++- src/disco/topo/fd_topo.h | 7 +++++++ src/discof/backtest/fd_backtest_tile.c | 19 ++++++++++++++++++- src/discof/replay/fd_replay_tile.c | 21 ++++++++++++++++++++- src/discof/replay/fd_replay_tile_private.h | 9 +++++++++ 10 files changed, 106 insertions(+), 4 deletions(-) diff --git a/src/app/firedancer-dev/commands/backtest.c b/src/app/firedancer-dev/commands/backtest.c index 6f3c31f39de..ca45534af60 100644 --- a/src/app/firedancer-dev/commands/backtest.c +++ b/src/app/firedancer-dev/commands/backtest.c @@ -450,6 +450,12 @@ backtest_topo( config_t * config ) { for( ulong i = 0; i < tile->replay.enable_features_cnt; i++ ) { fd_cstr_ncpy( tile->replay.enable_features[i], config->tiles.replay.enable_features[i], sizeof(tile->replay.enable_features[i]) ); } + + /* In a backtest the backt tile owns [development.force_mismatch] + disarm replay's copy so footer check doesn't force mismatch. */ + tile->replay.force_mismatch = 0; + tile->replay.force_mismatch_at_slot = 0UL; + tile->replay.force_mismatch_after = 0UL; } } diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index 780b0d12050..ddd0b4222a4 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -2339,6 +2339,25 @@ telemetry = true # Delays rooting by the given number of slots. root_distance = 32 + # Deliberately force a bank hash mismatch, for exercising the + # mismatch-handling paths. In a 'firedancer-dev backtest' run the + # backtest tile forces its ledger comparison to mismatch (the run + # logs the mismatch and exits); on a live/network run the replay + # tile forces the Alpenglow footer comparison to mismatch once (the + # block is marked dead and the validator keeps running). + [development.force_mismatch] + # Master switch. When false, nothing below has any effect. + enabled = false + + # When non-zero, force the mismatch on this absolute slot number. + at_slot = 0 + + # When non-zero, force the mismatch after this many slots have + # been replayed. With both at_slot and after left zero, the + # mismatch is forced on the first slot replayed. Setting both + # at_slot and after is rejected as ambiguous. + after = 0 + # Tuning for the accounts database. These options are for advanced # users and should not be changed in production. [development.accdb] diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index 20378c4a74e..ce858d902c3 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -1615,6 +1615,10 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->replay.dump_block_to_pb = config->capture.dump_block_to_pb; tile->replay.report_runtime_diffs = config->development.event.report_runtime_diffs; + tile->replay.force_mismatch = config->firedancer.development.force_mismatch.enabled; + tile->replay.force_mismatch_at_slot = config->firedancer.development.force_mismatch.at_slot; + tile->replay.force_mismatch_after = config->firedancer.development.force_mismatch.after; + if( FD_UNLIKELY( config->tiles.bundle.enabled ) ) { #define PARSE_BUNDLE_PUBKEY( _tile, f ) \ if( FD_UNLIKELY( !fd_base58_decode_32( config->tiles.bundle.f, tile->_tile.bundle.f ) ) ) \ @@ -1870,6 +1874,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, fd_cstr_ncpy( tile->backtest.ledger_path, config->firedancer.development.ledger_input.path, PATH_MAX ); tile->backtest.end_slot = config->firedancer.development.ledger_input.end_slot; tile->backtest.boot_timestamp_nanos = config->boot_timestamp_nanos; + tile->backtest.force_mismatch = config->firedancer.development.force_mismatch.enabled; + tile->backtest.force_mismatch_at_slot = config->firedancer.development.force_mismatch.at_slot; + tile->backtest.force_mismatch_after = config->firedancer.development.force_mismatch.after; } else if( FD_UNLIKELY( !strcmp( tile->name, "bundle" ) ) ) { fd_cstr_ncpy( tile->bundle.url, config->tiles.bundle.url, sizeof(tile->bundle.url) ); diff --git a/src/app/shared/fd_config.h b/src/app/shared/fd_config.h index 86a4126188f..6e85b3273c7 100644 --- a/src/app/shared/fd_config.h +++ b/src/app/shared/fd_config.h @@ -115,6 +115,12 @@ struct fd_configf { char affinity[ AFFINITY_SZ ]; ulong root_distance; } backtest; + + struct { + int enabled; + ulong at_slot; + ulong after; + } force_mismatch; } development; struct { diff --git a/src/app/shared/fd_config_json.c b/src/app/shared/fd_config_json.c index 56129762d1a..2fee2b6dedb 100644 --- a/src/app/shared/fd_config_json.c +++ b/src/app/shared/fd_config_json.c @@ -12,7 +12,7 @@ or knowingly skipped) before the constant is bumped. String keys of the user's own file are separately forced through the classification lists below. */ -FD_STATIC_ASSERT( sizeof(fd_config_t)==24458368UL, update_fd_config_to_json_for_the_layout_change ); +FD_STATIC_ASSERT( sizeof(fd_config_t)==24458392UL, update_fd_config_to_json_for_the_layout_change ); #define REDACTED "[redacted]" @@ -488,6 +488,11 @@ fd_config_to_json( fd_config_t const * config, jw_str ( &w, "affinity", f->development.backtest.affinity ); jw_ulong( &w, "root_distance", f->development.backtest.root_distance ); jw_obj_close( &w ); + jw_obj_open( &w, "force_mismatch" ); + jw_bool ( &w, "enabled", f->development.force_mismatch.enabled ); + jw_ulong( &w, "at_slot", f->development.force_mismatch.at_slot ); + jw_ulong( &w, "after", f->development.force_mismatch.after ); + jw_obj_close( &w ); jw_obj_open( &w, "bench" ); jw_ulong( &w, "benchg_tile_count", config->development.bench.benchg_tile_count ); jw_ulong( &w, "benchs_tile_count", config->development.bench.benchs_tile_count ); diff --git a/src/app/shared/fd_config_parse.c b/src/app/shared/fd_config_parse.c index ad342d3494a..0c27cefa0c6 100644 --- a/src/app/shared/fd_config_parse.c +++ b/src/app/shared/fd_config_parse.c @@ -5,7 +5,10 @@ static void fd_config_check_configf( fd_config_t * config, fd_configf_t * config_f ) { - (void)config_f; + if( FD_UNLIKELY( config_f->development.force_mismatch.at_slot && + config_f->development.force_mismatch.after ) ) { + FD_LOG_ERR(( "[development.force_mismatch] set both at_slot and after; they are mutually exclusive" )); + } if( FD_UNLIKELY( strlen( config->paths.snapshots )>PATH_MAX-1UL ) ) { FD_LOG_ERR(( "[config->paths.snapshots] is too long (max %lu)", PATH_MAX-1UL )); } @@ -93,6 +96,10 @@ fd_config_extract_podf( uchar * pod, CFG_POP ( cstr, development.backtest.affinity ); CFG_POP ( ulong, development.backtest.root_distance ); + CFG_POP ( bool, development.force_mismatch.enabled ); + CFG_POP ( ulong, development.force_mismatch.at_slot ); + CFG_POP ( ulong, development.force_mismatch.after ); + return config; } diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index d08e5cde21b..70bdd936a93 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -530,6 +530,10 @@ struct fd_topo_tile { } bundle; int alpenglow; + + int force_mismatch; + ulong force_mismatch_at_slot; + ulong force_mismatch_after; } replay; struct { @@ -614,6 +618,9 @@ struct fd_topo_tile { ulong end_slot; ulong root_distance; long boot_timestamp_nanos; + int force_mismatch; + ulong force_mismatch_at_slot; + ulong force_mismatch_after; } backtest; struct { diff --git a/src/discof/backtest/fd_backtest_tile.c b/src/discof/backtest/fd_backtest_tile.c index a8eeabc79e7..8a5bcd50981 100644 --- a/src/discof/backtest/fd_backtest_tile.c +++ b/src/discof/backtest/fd_backtest_tile.c @@ -90,6 +90,10 @@ struct fd_backt_tile { long publish_time; ulong slot_cnt; + int force_mismatch; /* deliberately force a bank hash mismatch */ + ulong force_mismatch_at_slot; /* 0 = unset: force on a specific slot number */ + ulong force_mismatch_after; /* 0 = unset: force after this many slots replayed */ + long boot_time; long snapshot_done_time; long replay_ready_time; @@ -470,7 +474,16 @@ returnable_frag( fd_backt_tile_t * ctx, if( FD_UNLIKELY( !fd_backtest_src_slot_info( ctx->src, &slot_info, msg->slot ) || !slot_info.bank_hash_set ) ) { FD_LOG_ERR(( "No bank hash available for slot %lu", msg->slot )); } - if( FD_LIKELY( !memcmp( msg->bank_hash.uc, slot_info.bank_hash.uc, 32UL ) ) ) { + /* Deliberately force a mismatch for the configured slot so the + mismatch path is exercised. ctx->slot_cnt is the number of + slots completed before this one, so "first slot" is + slot_cnt==0 and "after N" is slot_cnt==N. */ + int force = ctx->force_mismatch && + ( ( ctx->force_mismatch_at_slot && msg->slot==ctx->force_mismatch_at_slot ) || + ( ctx->force_mismatch_after && ctx->slot_cnt==ctx->force_mismatch_after ) || + ( !ctx->force_mismatch_at_slot && !ctx->force_mismatch_after && !ctx->slot_cnt ) ); + if( FD_UNLIKELY( force ) ) FD_LOG_WARNING(( "forcing a bank hash mismatch at slot %lu (%s)", msg->slot, bh_got_b58 )); + if( FD_LIKELY( !force && !memcmp( msg->bank_hash.uc, slot_info.bank_hash.uc, 32UL ) ) ) { FD_LOG_NOTICE(( "Bank hash matches! slot=%lu, hash=%-44s (switch %.2f ms, begin %.2f ms, exec %6.2f ms, finish %.2f ms)", msg->slot, bh_got_b58, (double)(msg->preparation_begin_nanos-prior_completion_timestamp)/1e6, (double)(msg->first_transaction_scheduled_nanos-msg->preparation_begin_nanos)/1e6, @@ -623,6 +636,10 @@ unprivileged_init( fd_topo_t const * topo, ctx->root_distance = tile->backtest.root_distance; ctx->boot_time = tile->backtest.boot_timestamp_nanos; + ctx->force_mismatch = tile->backtest.force_mismatch; + ctx->force_mismatch_at_slot = tile->backtest.force_mismatch_at_slot; + ctx->force_mismatch_after = tile->backtest.force_mismatch_after; + ctx->src = NULL; if( FD_LIKELY( tile->backtest.ledger_path[ 0 ] ) ) { /* empty path: load the snapshot, then exit */ fd_backtest_src_opts_t opts = { diff --git a/src/discof/replay/fd_replay_tile.c b/src/discof/replay/fd_replay_tile.c index 93b5ff19262..f74c861fc18 100644 --- a/src/discof/replay/fd_replay_tile.c +++ b/src/discof/replay/fd_replay_tile.c @@ -1791,7 +1791,20 @@ replay_block_complete( fd_replay_tile_t * ctx, ulong execution_fees_pre_settle, ulong priority_fees_pre_settle ) { fd_hash_t const * footer_bank_hash = &footer->bank_hash; - if( FD_UNLIKELY( memcmp( footer_bank_hash->uc, bank->f.bank_hash.uc, sizeof(fd_hash_t) ) ) ) { + /* Deliberately force the footer comparison to mismatch once, to + exercise the dead-block path. ctx->blocks_completed is the count + of blocks that completed this check before this one, so "first + block" is blocks_completed==0 and "after N" is blocks_completed==N. */ + int force = ctx->force_mismatch && !ctx->force_mismatch_fired && + ( ( ctx->force_mismatch_at_slot && bank->f.slot==ctx->force_mismatch_at_slot ) || + ( ctx->force_mismatch_after && ctx->blocks_completed==ctx->force_mismatch_after ) || + ( !ctx->force_mismatch_at_slot && !ctx->force_mismatch_after && !ctx->blocks_completed ) ); + ctx->blocks_completed++; + if( FD_UNLIKELY( force ) ) { + ctx->force_mismatch_fired = 1; + FD_LOG_WARNING(( "slot %lu: forcing a footer bank hash mismatch", bank->f.slot )); + } + if( FD_UNLIKELY( force || memcmp( footer_bank_hash->uc, bank->f.bank_hash.uc, sizeof(fd_hash_t) ) ) ) { FD_BASE58_ENCODE_32_BYTES( footer_bank_hash->uc, footer_bank_hash_b58 ); FD_BASE58_ENCODE_32_BYTES( bank->f.bank_hash.uc, executed_bank_hash_b58 ); FD_LOG_WARNING(( "slot %lu: bank hash mismatch, footer declares %s but executed %s. ", bank->f.slot, footer_bank_hash_b58, executed_bank_hash_b58 )); @@ -4768,6 +4781,12 @@ unprivileged_init( fd_topo_t const * topo, ctx->report_runtime_diffs = tile->replay.report_runtime_diffs; ctx->banks->report_runtime_diffs = tile->replay.report_runtime_diffs; + ctx->force_mismatch = tile->replay.force_mismatch; + ctx->force_mismatch_at_slot = tile->replay.force_mismatch_at_slot; + ctx->force_mismatch_after = tile->replay.force_mismatch_after; + ctx->blocks_completed = 0UL; + ctx->force_mismatch_fired = 0; + ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() ); if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) ) FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) )); diff --git a/src/discof/replay/fd_replay_tile_private.h b/src/discof/replay/fd_replay_tile_private.h index 422f7239643..fb7ec213ba4 100644 --- a/src/discof/replay/fd_replay_tile_private.h +++ b/src/discof/replay/fd_replay_tile_private.h @@ -644,6 +644,15 @@ struct fd_replay_tile { /* If non-zero, emit the runtime events during replay. */ int report_runtime_diffs; + + /* Deliberately force an Alpenglow footer bank hash mismatch once, to + exercise the dead-block path. force_mismatch_fired latches after + the single forced mismatch so it never fires again. */ + int force_mismatch; + ulong force_mismatch_at_slot; /* 0 = unset: force on a specific slot number */ + ulong force_mismatch_after; /* 0 = unset: force after this many blocks completed */ + ulong blocks_completed; /* count of completed blocks, for force_mismatch_after */ + int force_mismatch_fired; /* latched after the forced mismatch */ }; typedef struct fd_replay_tile fd_replay_tile_t;