dev: add [development.force_mismatch] to force a bank hash mismatch

src/app/firedancer-dev/commands/backtest.c

@@ -450,6 +450,12 @@ backtest_topo( config_t * config ) {
       for( ulong i = 0; i < tile->replay.enable_features_cnt; i++ ) {
         fd_cstr_ncpy( tile->replay.enable_features[i], config->tiles.replay.enable_features[i], sizeof(tile->replay.enable_features[i]) );
       }
+
+      /* In a backtest the backt tile owns [development.force_mismatch]
+         disarm replay's copy so footer check doesn't force mismatch. */
+      tile->replay.force_mismatch         = 0;
+      tile->replay.force_mismatch_at_slot = 0UL;
+      tile->replay.force_mismatch_after   = 0UL;
     }
   }
 

src/app/firedancer/config/default.toml

@@ -2339,6 +2339,25 @@ telemetry = true
         # Delays rooting by the given number of slots.
         root_distance = 32
 
+    # Deliberately force a bank hash mismatch, for exercising the
+    # mismatch-handling paths.  In a 'firedancer-dev backtest' run the
+    # backtest tile forces its ledger comparison to mismatch (the run
+    # logs the mismatch and exits); on a live/network run the replay
+    # tile forces the Alpenglow footer comparison to mismatch once (the
+    # block is marked dead and the validator keeps running).
+    [development.force_mismatch]
+        # Master switch.  When false, nothing below has any effect.
+        enabled = false
+
+        # When non-zero, force the mismatch on this absolute slot number.
+        at_slot = 0
+
+        # When non-zero, force the mismatch after this many slots have
+        # been replayed.  With both at_slot and after left zero, the
+        # mismatch is forced on the first slot replayed.  Setting both
+        # at_slot and after is rejected as ambiguous.
+        after = 0
+
     # Tuning for the accounts database.  These options are for advanced
     # users and should not be changed in production.
     [development.accdb]

src/app/firedancer/topology.c

@@ -1615,6 +1615,10 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
     tile->replay.dump_block_to_pb = config->capture.dump_block_to_pb;
     tile->replay.report_runtime_diffs = config->development.event.report_runtime_diffs;
 
+    tile->replay.force_mismatch         = config->firedancer.development.force_mismatch.enabled;
+    tile->replay.force_mismatch_at_slot = config->firedancer.development.force_mismatch.at_slot;
+    tile->replay.force_mismatch_after   = config->firedancer.development.force_mismatch.after;
+
     if( FD_UNLIKELY( config->tiles.bundle.enabled ) ) {
 #define PARSE_BUNDLE_PUBKEY( _tile, f ) \
       if( FD_UNLIKELY( !fd_base58_decode_32( config->tiles.bundle.f, tile->_tile.bundle.f ) ) )  \
@@ -1870,6 +1874,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile,
     fd_cstr_ncpy( tile->backtest.ledger_path, config->firedancer.development.ledger_input.path, PATH_MAX );
     tile->backtest.end_slot = config->firedancer.development.ledger_input.end_slot;
     tile->backtest.boot_timestamp_nanos = config->boot_timestamp_nanos;
+    tile->backtest.force_mismatch         = config->firedancer.development.force_mismatch.enabled;
+    tile->backtest.force_mismatch_at_slot = config->firedancer.development.force_mismatch.at_slot;
+    tile->backtest.force_mismatch_after   = config->firedancer.development.force_mismatch.after;
 
   } else if( FD_UNLIKELY( !strcmp( tile->name, "bundle" ) ) ) {
     fd_cstr_ncpy( tile->bundle.url, config->tiles.bundle.url, sizeof(tile->bundle.url) );

src/app/shared/fd_config.h

@@ -115,6 +115,12 @@ struct fd_configf {
       char  affinity[ AFFINITY_SZ ];
       ulong root_distance;
     } backtest;
+
+    struct {
+      int   enabled;
+      ulong at_slot;
+      ulong after;
+    } force_mismatch;
   } development;
 
   struct {

src/app/shared/fd_config_json.c

@@ -12,7 +12,7 @@
    or knowingly skipped) before the constant is bumped.  String keys of
    the user's own file are separately forced through the classification
    lists below. */
-FD_STATIC_ASSERT( sizeof(fd_config_t)==24458368UL, update_fd_config_to_json_for_the_layout_change );
+FD_STATIC_ASSERT( sizeof(fd_config_t)==24458392UL, update_fd_config_to_json_for_the_layout_change );
 
 #define REDACTED "[redacted]"
 
@@ -488,6 +488,11 @@ fd_config_to_json( fd_config_t const * config,
       jw_str  ( &w, "affinity",      f->development.backtest.affinity );
       jw_ulong( &w, "root_distance", f->development.backtest.root_distance );
     jw_obj_close( &w );
+    jw_obj_open( &w, "force_mismatch" );
+      jw_bool ( &w, "enabled", f->development.force_mismatch.enabled );
+      jw_ulong( &w, "at_slot", f->development.force_mismatch.at_slot );
+      jw_ulong( &w, "after",   f->development.force_mismatch.after   );
+    jw_obj_close( &w );
     jw_obj_open( &w, "bench" );
       jw_ulong( &w, "benchg_tile_count",            config->development.bench.benchg_tile_count );
       jw_ulong( &w, "benchs_tile_count",            config->development.bench.benchs_tile_count );

src/app/shared/fd_config_parse.c

@@ -5,7 +5,10 @@
 static void
 fd_config_check_configf( fd_config_t *  config,
                          fd_configf_t * config_f ) {
-  (void)config_f;
+  if( FD_UNLIKELY( config_f->development.force_mismatch.at_slot &&
+                   config_f->development.force_mismatch.after ) ) {
+    FD_LOG_ERR(( "[development.force_mismatch] set both at_slot and after; they are mutually exclusive" ));
+  }
   if( FD_UNLIKELY( strlen( config->paths.snapshots )>PATH_MAX-1UL ) ) {
     FD_LOG_ERR(( "[config->paths.snapshots] is too long (max %lu)", PATH_MAX-1UL ));
   }
@@ -93,6 +96,10 @@ fd_config_extract_podf( uchar *        pod,
   CFG_POP      ( cstr,   development.backtest.affinity                       );
   CFG_POP      ( ulong,  development.backtest.root_distance                  );
 
+  CFG_POP      ( bool,   development.force_mismatch.enabled                  );
+  CFG_POP      ( ulong,  development.force_mismatch.at_slot                  );
+  CFG_POP      ( ulong,  development.force_mismatch.after                    );
+
 
   return config;
 }

src/disco/topo/fd_topo.h

@@ -530,6 +530,10 @@ struct fd_topo_tile {
       } bundle;
 
       int alpenglow;
+
+      int   force_mismatch;
+      ulong force_mismatch_at_slot;
+      ulong force_mismatch_after;
     } replay;
 
     struct {
@@ -614,6 +618,9 @@ struct fd_topo_tile {
       ulong end_slot;
       ulong root_distance;
       long  boot_timestamp_nanos;
+      int   force_mismatch;
+      ulong force_mismatch_at_slot;
+      ulong force_mismatch_after;
     } backtest;
 
     struct {

src/discof/backtest/fd_backtest_tile.c

@@ -90,6 +90,10 @@ struct fd_backt_tile {
   long  publish_time;
   ulong slot_cnt;
 
+  int   force_mismatch;         /* deliberately force a bank hash mismatch */
+  ulong force_mismatch_at_slot; /* 0 = unset: force on a specific slot number */
+  ulong force_mismatch_after;   /* 0 = unset: force after this many slots replayed */
+
   long boot_time;
   long snapshot_done_time;
   long replay_ready_time;
@@ -470,7 +474,16 @@ returnable_frag( fd_backt_tile_t *   ctx,
       if( FD_UNLIKELY( !fd_backtest_src_slot_info( ctx->src, &slot_info, msg->slot ) || !slot_info.bank_hash_set ) ) {
         FD_LOG_ERR(( "No bank hash available for slot %lu", msg->slot ));
       }
-      if( FD_LIKELY( !memcmp( msg->bank_hash.uc, slot_info.bank_hash.uc, 32UL ) ) ) {
+      /* Deliberately force a mismatch for the configured slot so the
+         mismatch path is exercised. ctx->slot_cnt is the number of
+         slots completed before this one, so "first slot" is
+         slot_cnt==0 and "after N" is slot_cnt==N. */
+      int force = ctx->force_mismatch &&
+                  ( ( ctx->force_mismatch_at_slot && msg->slot==ctx->force_mismatch_at_slot ) ||
+                    ( ctx->force_mismatch_after   && ctx->slot_cnt==ctx->force_mismatch_after ) ||
+                    ( !ctx->force_mismatch_at_slot && !ctx->force_mismatch_after && !ctx->slot_cnt ) );
+      if( FD_UNLIKELY( force ) ) FD_LOG_WARNING(( "forcing a bank hash mismatch at slot %lu (%s)", msg->slot, bh_got_b58 ));
+      if( FD_LIKELY( !force && !memcmp( msg->bank_hash.uc, slot_info.bank_hash.uc, 32UL ) ) ) {
         FD_LOG_NOTICE(( "Bank hash matches! slot=%lu, hash=%-44s (switch %.2f ms, begin %.2f ms, exec %6.2f ms, finish %.2f ms)", msg->slot, bh_got_b58,
           (double)(msg->preparation_begin_nanos-prior_completion_timestamp)/1e6,
           (double)(msg->first_transaction_scheduled_nanos-msg->preparation_begin_nanos)/1e6,
@@ -623,6 +636,10 @@ unprivileged_init( fd_topo_t const *      topo,
   ctx->root_distance = tile->backtest.root_distance;
   ctx->boot_time     = tile->backtest.boot_timestamp_nanos;
 
+  ctx->force_mismatch         = tile->backtest.force_mismatch;
+  ctx->force_mismatch_at_slot = tile->backtest.force_mismatch_at_slot;
+  ctx->force_mismatch_after   = tile->backtest.force_mismatch_after;
+
   ctx->src = NULL;
   if( FD_LIKELY( tile->backtest.ledger_path[ 0 ] ) ) { /* empty path: load the snapshot, then exit */
     fd_backtest_src_opts_t opts = {

src/discof/replay/fd_replay_tile.c

@@ -1791,7 +1791,20 @@ replay_block_complete( fd_replay_tile_t *        ctx,
                        ulong                     execution_fees_pre_settle,
                        ulong                     priority_fees_pre_settle ) {
   fd_hash_t const * footer_bank_hash = &footer->bank_hash;
-  if( FD_UNLIKELY( memcmp( footer_bank_hash->uc, bank->f.bank_hash.uc, sizeof(fd_hash_t) ) ) ) {
+  /* Deliberately force the footer comparison to mismatch once, to
+     exercise the dead-block path.  ctx->blocks_completed is the count
+     of blocks that completed this check before this one, so "first
+     block" is blocks_completed==0 and "after N" is blocks_completed==N. */
+  int force = ctx->force_mismatch && !ctx->force_mismatch_fired &&
+              ( ( ctx->force_mismatch_at_slot && bank->f.slot==ctx->force_mismatch_at_slot ) ||
+                ( ctx->force_mismatch_after   && ctx->blocks_completed==ctx->force_mismatch_after ) ||
+                ( !ctx->force_mismatch_at_slot && !ctx->force_mismatch_after && !ctx->blocks_completed ) );
+  ctx->blocks_completed++;
+  if( FD_UNLIKELY( force ) ) {
+    ctx->force_mismatch_fired = 1;
+    FD_LOG_WARNING(( "slot %lu: forcing a footer bank hash mismatch", bank->f.slot ));
+  }
+  if( FD_UNLIKELY( force || memcmp( footer_bank_hash->uc, bank->f.bank_hash.uc, sizeof(fd_hash_t) ) ) ) {
     FD_BASE58_ENCODE_32_BYTES( footer_bank_hash->uc,   footer_bank_hash_b58   );
     FD_BASE58_ENCODE_32_BYTES( bank->f.bank_hash.uc, executed_bank_hash_b58 );
     FD_LOG_WARNING(( "slot %lu: bank hash mismatch, footer declares %s but executed %s. ", bank->f.slot, footer_bank_hash_b58, executed_bank_hash_b58 ));
@@ -4768,6 +4781,12 @@ unprivileged_init( fd_topo_t const *      topo,
   ctx->report_runtime_diffs = tile->replay.report_runtime_diffs;
   ctx->banks->report_runtime_diffs = tile->replay.report_runtime_diffs;
 
+  ctx->force_mismatch         = tile->replay.force_mismatch;
+  ctx->force_mismatch_at_slot = tile->replay.force_mismatch_at_slot;
+  ctx->force_mismatch_after   = tile->replay.force_mismatch_after;
+  ctx->blocks_completed       = 0UL;
+  ctx->force_mismatch_fired   = 0;
+
   ulong scratch_top = FD_SCRATCH_ALLOC_FINI( l, scratch_align() );
   if( FD_UNLIKELY( scratch_top > (ulong)scratch + scratch_footprint( tile ) ) )
     FD_LOG_ERR(( "scratch overflow %lu %lu %lu", scratch_top - (ulong)scratch - scratch_footprint( tile ), scratch_top, (ulong)scratch + scratch_footprint( tile ) ));

src/discof/replay/fd_replay_tile_private.h

@@ -644,6 +644,15 @@ struct fd_replay_tile {
 
   /* If non-zero, emit the runtime events during replay. */
   int report_runtime_diffs;
+
+  /* Deliberately force an Alpenglow footer bank hash mismatch once, to
+     exercise the dead-block path.  force_mismatch_fired latches after
+     the single forced mismatch so it never fires again. */
+  int   force_mismatch;
+  ulong force_mismatch_at_slot; /* 0 = unset: force on a specific slot number */
+  ulong force_mismatch_after;   /* 0 = unset: force after this many blocks completed */
+  ulong blocks_completed;       /* count of completed blocks, for force_mismatch_after */
+  int   force_mismatch_fired;   /* latched after the forced mismatch */
 };
 
 typedef struct fd_replay_tile fd_replay_tile_t;