diff --git a/book/api/cli.md b/book/api/cli.md index bbaba117e07..7f0441e16ff 100644 --- a/book/api/cli.md +++ b/book/api/cli.md @@ -190,6 +190,16 @@ argument is required and must be the path to an Agave style `identity.json` keypair file. If the path is specified as `-` the key will instead be read from `stdin`. +::: tip NOTE + +Between two Firedancer validators, Firedancer supports a native +[failover](#failover-promote) mechanism that is faster and easier than +`set-identity`, and using `set-identity` between them is heavily +discouraged. Use `set-identity` primarily for failing over between Agave +and Firedancer. + +::: + It is not generally safe to call `set-identity`, as another validator might be running with the same identity, and if they both produce a block or vote concurrently, the validator may violate consensus and be @@ -333,6 +343,150 @@ than 16). <<< @/snippets/commands/add-authorized-voter.ansi +## `failover promote` +This command takes an identity key, with its vote history, from the +remote validator currently running it and installs it on this local +validator. Run this command on the machine that should take over, +with the target keypair of the identity to take. The machine that +originally ran it will switch to a random junk key that it generated +at boot. The validators must use the same vote account and have +failover enabled, which is on by default. + +```sh +firedancer failover promote [] [--name ] [--yes] [--force] [--address
] [--port ] +``` + +The command attaches to the local running validator and discovers it +automatically. If more than one is running, pass `--name ` to +select one (see [`ps`](#ps)). It asks for confirmation when run from a +terminal unless `--yes` is given. + +The local validator finds the address of the machine running the +identity through gossip. Use `--address` to give its IPv4 address or host +name instead, for example when connecting over a private network. Use +`--port` if its failover listener does not use the default port 8010. + +For example, to connect to a known failover listener over a private +network, run the following on the machine that should take over. + +```sh +firedancer failover promote /path/to/identity.json --address 192.0.2.10 --port 8010 +``` + +### Setup + +Failover runs on every validator with `[failover.enabled]` set to `true`. +Each validator boots under its own `[paths.identity_key]`, and all must +use the same vote account. The validator that should vote keeps the +identity's keypair there. On every other machine, create a random unstaked +placeholder keypair of its own and point `[paths.identity_key]` at it, so +it replays without voting until it takes over. + +```sh +firedancer keys new /home/firedancer/junk.json --config config.toml +``` + +Alternatively, set `[failover.boot_to_standby]` to `true` on the other +machine and keep the identity's keypair in its `[paths.identity_key]`. +It then boots as a standby under an unstaked junk key generated in memory +at each start, and loads the keypair in `[paths.identity_key]` as the +identity it takes, so `failover promote` there reads the keypair from the +configuration instead of a command line argument. The junk private key +is ephemeral and not written to a file. This requires `[failover.enabled]` +and `[paths.vote_account]`. + +To hand the identity over, run the following on the machine that should +take it, with the keypair of the identity to take. + +```sh +firedancer failover promote /path/to/identity.json +firedancer failover status +``` + +For example, an accepted request prints the handoff ID and the status +needed to confirm completion. + +<<< @/snippets/commands/failover-promote.ansi + +The machine that gave it up then runs its junk key. To take the identity +back, run `failover promote` again on the demoted machine. + +### Restarts + +A validator boots under its `[paths.identity_key]` unless +`[failover.boot_to_standby]` is set. Before restarting the machine that +gave the identity up, point its `[paths.identity_key]` at a junk key of +its own, or set `[failover.boot_to_standby]` there. If it boots under the +identity while the other machine runs it, the duplicate instance check +stops the other machine. A machine with `[failover.boot_to_standby]` boots +under a fresh junk key every time, so it can restart while the other +machine runs the identity, and it takes the identity back with +`failover promote`. + +Leaving an identity renames its vote history file in +`[paths.vote_history]` to `vote_history-.bin.bak`, and a +validator refuses to boot under a key whose vote history file is missing +while other vote history files are there. + +- To restart a machine that took the identity under its junk key, first + rename `vote_history-.bin.bak` back to `vote_history-.bin`. +- To restart the machine that gave the identity up under a junk key, + first move the `vote_history-*` files out of `[paths.vote_history]`. + The identity's vote history now lives on the other machine. + +### Forced promotion + +With `--force`, the local validator takes the identity without +contacting the machine running it. Use it when no machine runs the +identity, for example after both restarted, or when the other machine +cannot sign. + +::: warning WARNING + +Only use `--force` after ensuring every other machine with the identity +cannot sign. It skips the checks on the other machine and can leave votes +and leader windows unaccounted for by accepting incomplete or empty history. +Never force promotion on both machines at once. + +::: + +| Arguments | Description | +|-----------------------|-------------| +| `` | The keypair file of the identity to take. +| `--name ` | Select the validator instance to attach to if more than one is running on this host. | +| `--config ` | Optionally use the configuration TOML file the validator was started with. | +| `--yes` | Skip the confirmation prompt. | +| `--force` | Forcibly take the identity without cooperation with the machine running it, allowing incomplete or empty vote history. | +| `--address
` | Use this IPv4 address or host name for the machine running the identity instead of the address found through gossip. | +| `--port ` | Set the failover TCP port of the machine running the identity. The default is 8010. | + +## `failover status` +This command prints the local running validator's failover status. Use +it to check progress after [`failover promote`](#failover-promote). + +```sh +firedancer failover status [--name ] +``` + +The command discovers the local running validator automatically. If more +than one is running, pass `--name ` to select one (see [`ps`](#ps)). +It exits with code 0 if status was retrieved, including when failover +is disabled. Otherwise it exits with a non-zero code and prints the +reason to `stderr`. + +The following example shows a completed handoff. The last handoff ID +matches the ID printed by `failover promote` above. The connection closes +after completion, so `peer runs target identity` is unknown until another +authenticated session is established. The machine that handed over its +identity reconnects once it is discoverable through gossip. + +<<< @/snippets/commands/failover-status.ansi + +| Arguments | Description | +|-----------------------|-------------| +| `--name ` | Select the validator instance to attach to if more than one is running on this host. | +| `--config ` | Optionally use the configuration TOML file the validator was started with. | + ## `remove-all-authorized-voters` Removes all authorized voters from the running validator, including any seeded from `[paths.authorized_voter_paths]` at startup as well as any diff --git a/book/snippets/commands/failover-promote.ansi b/book/snippets/commands/failover-promote.ansi new file mode 100644 index 00000000000..b8bfd93a00e --- /dev/null +++ b/book/snippets/commands/failover-promote.ansi @@ -0,0 +1,5 @@ +$ firedancer failover promote /home/firedancer/identity.json --yes +failover promote accepted +runs target identity: no +action: taking the identity, requesting the final vote state +handoff: 123456789, done when `failover status` here shows runs target identity: yes and last handoff: 123456789, taken diff --git a/book/snippets/commands/failover-status.ansi b/book/snippets/commands/failover-status.ansi new file mode 100644 index 00000000000..dc7b0e5a851 --- /dev/null +++ b/book/snippets/commands/failover-status.ansi @@ -0,0 +1,16 @@ +$ firedancer failover status +failover: enabled +target identity: 2vJXGJVGTxabY97CtBfby7KcHz3Pb2ZEX8aH6vLbqvTf +installed identity: 2vJXGJVGTxabY97CtBfby7KcHz3Pb2ZEX8aH6vLbqvTf (the target) +runs target identity: yes +junk key: LxRHL4cEBPTKxv5HgoMp82kEY9cqwJndsEaEp7ZXswL +action: idle +stuck: no +link: listening +peer runs target identity: unknown, no authenticated failover session +peer boot id: 0000000000000001 +peer address: 192.0.2.10:8010, from gossip +last handoff: 123456789, taken +other holder: N/A, this machine runs the identity +takeover adopts: an empty history (requires --force) +coverage floor: slot 300 diff --git a/src/app/firedancer-dev/main.h b/src/app/firedancer-dev/main.h index cd0bffd2b64..e0ab2d3670f 100644 --- a/src/app/firedancer-dev/main.h +++ b/src/app/firedancer-dev/main.h @@ -117,6 +117,7 @@ extern fd_topo_run_tile_t fd_tile_udpecho; extern fd_topo_run_tile_t fd_tile_genesi; extern fd_topo_run_tile_t fd_tile_ipecho; extern fd_topo_run_tile_t fd_tile_admin; +extern fd_topo_run_tile_t fd_tile_failov; extern fd_topo_run_tile_t fd_tile_gossvf; extern fd_topo_run_tile_t fd_tile_gossip; @@ -185,6 +186,7 @@ fd_topo_run_tile_t * TILES[] = { &fd_tile_genesi, &fd_tile_ipecho, &fd_tile_admin, + &fd_tile_failov, &fd_tile_solcap, &fd_tile_snapmk, &fd_tile_snapzp, @@ -203,6 +205,7 @@ extern action_t fd_action_mem; extern action_t fd_action_netconf; extern action_t fd_action_set_identity; extern action_t fd_action_get_identity; +extern action_t fd_action_failover; extern action_t fd_action_ps; extern action_t fd_action_version; extern action_t fd_action_bench; @@ -244,6 +247,7 @@ action_t * ACTIONS[] = { &fd_action_netconf, &fd_action_set_identity, &fd_action_get_identity, + &fd_action_failover, &fd_action_ps, &fd_action_help, &fd_action_metrics, diff --git a/src/app/firedancer/Local.mk b/src/app/firedancer/Local.mk index 26896862d3b..91fd45f0415 100644 --- a/src/app/firedancer/Local.mk +++ b/src/app/firedancer/Local.mk @@ -24,6 +24,7 @@ $(call add-objs,commands/shred_version,fd_firedancer) $(call add-objs,commands/snapshot_create,fd_firedancer) $(call add-objs,commands/set_identity,fd_firedancer) $(call add-objs,commands/get_identity,fd_firedancer) +$(call add-objs,commands/failover,fd_firedancer) $(call add-objs,commands/adminctl_client,fd_firedancer) $(call add-objs,commands/monitor_gossip/monitor_gossip commands/monitor_gossip/gossip_diag,fd_firedancer) $(call add-objs,commands/wait,fd_firedancer) diff --git a/src/app/firedancer/commands/failover.c b/src/app/firedancer/commands/failover.c new file mode 100644 index 00000000000..87657ff590a --- /dev/null +++ b/src/app/firedancer/commands/failover.c @@ -0,0 +1,504 @@ +#define _GNU_SOURCE +#include "adminctl_client.h" +#include "../../shared/fd_config.h" +#include "../../shared/fd_action.h" +#include "../../../discof/failover/fd_failover_proto.h" +#include "../../../disco/topo/fd_dns_resolve.h" +#include "../../../util/net/fd_ip4.h" +#include "../../../disco/keyguard/fd_keyload.h" +#include "../../../ballet/base58/fd_base58.h" +#include "../../../ballet/ed25519/fd_ed25519.h" +#include "../../platform/fd_cap_chk.h" + +#include +#include +#include + +/* human-readable output string lookup tables */ +static char const * const ACTION_NAMES[] = { + "idle", + "giving up the identity, switching to the junk key", + "giving up the identity, waiting for the peer's answer", + "taking the identity, waiting for replay", + "taking the identity, waiting for vote history adoption", + "taking the identity, switching to it", + "taking the identity, requesting the final vote state", + "taking the identity, waiting for the peer to record the handoff", + "giving up the identity, waiting for the vote history stream to drain" +}; +static char const * const SESSION_NAMES[] = { + "listening", "dialing", "hello", "paired", "backoff" +}; +static char const * const SOURCE_NAMES[] = { + "the stored peer vote history", "an empty history (requires --force)" +}; +static char const * const HANDOFF_NAMES[] = { + "nothing handed over", "pending", "taken", "declined", "restarted", "cancelled by `failover promote --force`", + "the dialed machine does not run the identity" +}; + +#define ACTION_NAME_CNT ( sizeof(ACTION_NAMES )/sizeof(ACTION_NAMES [ 0 ]) ) +#define SESSION_NAME_CNT ( sizeof(SESSION_NAMES)/sizeof(SESSION_NAMES[ 0 ]) ) +#define SOURCE_NAME_CNT ( sizeof(SOURCE_NAMES )/sizeof(SOURCE_NAMES [ 0 ]) ) +#define HANDOFF_NAME_CNT ( sizeof(HANDOFF_NAMES)/sizeof(HANDOFF_NAMES[ 0 ]) ) + +FD_STATIC_ASSERT( ACTION_NAME_CNT ==FD_FAILOVER_ACTION_CNT, action_names ); +FD_STATIC_ASSERT( SESSION_NAME_CNT==FD_FAILOVER_SESSION_CNT, session_names ); +FD_STATIC_ASSERT( SOURCE_NAME_CNT ==FD_FAILOVER_SOURCE_CNT, source_names ); +FD_STATIC_ASSERT( HANDOFF_NAME_CNT==FD_FAILOVER_HANDOFF_CNT, handoff_names ); + +static char const * +action_name( uchar action ) { + return actionfailover.name, name, sizeof(args->failover.name) ); + args->failover.yes = fd_env_strip_cmdline_contains( pargc, pargv, "--yes" ); + args->failover.force = fd_env_strip_cmdline_contains( pargc, pargv, "--force" ); + + args->failover.cmd = (int)( status ? FD_ADMINCTL_FAILOVER_CMD_STATUS : + args->failover.force ? FD_ADMINCTL_FAILOVER_CMD_PROMOTE : FD_ADMINCTL_FAILOVER_CMD_HANDOFF ); + + if( FD_UNLIKELY( args->failover.yes && args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_STATUS ) ) { + FD_LOG_ERR(( "--yes is only meaningful for `failover promote`" )); + } + if( FD_UNLIKELY( args->failover.force && args->failover.cmd!=(int)FD_ADMINCTL_FAILOVER_CMD_PROMOTE ) ) { + FD_LOG_ERR(( "--force is only meaningful for `failover promote`" )); + } + if( FD_UNLIKELY( ( address || port ) && args->failover.cmd!=(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) ) { + FD_LOG_ERR(( "--address and --port are only meaningful for `failover promote` without --force" )); + } + if( FD_UNLIKELY( address && !fd_dns_resolve_address( address, &args->failover.addr ) ) ) { + FD_LOG_ERR(( "could not resolve --address `%s` to an IPv4 address", address )); + } + if( FD_UNLIKELY( address && !args->failover.addr ) ) { + FD_LOG_ERR(( "--address `%s` is 0.0.0.0, provide the address of the machine running the identity", address )); + } + if( port ) { + uint value = 0U; + for( char const * c=port; *c; c++ ) { + uint digit = (uint)( *c-'0' ); + if( FD_UNLIKELY( digit>9U || value*10U+digit>65535U ) ) { + FD_LOG_ERR(( "--port `%s` must be a port number from 1 to 65535", port )); + } + value = value*10U+digit; + } + if( FD_UNLIKELY( !value ) ) { + FD_LOG_ERR(( "--port `%s` must be a port number from 1 to 65535", port )); + } + args->failover.port = (ushort)value; + } + if( !status && *pargc && (*pargv)[ 0 ][ 0 ]!='-' ) { + fd_cstr_ncpy( args->failover.keypair_path, (*pargv)[ 0 ], sizeof(args->failover.keypair_path) ); + (*pargc)--; + (*pargv)++; + } +} + +static void +failover_args( int * pargc, + char *** pargv, + args_t * args ) { + if( FD_UNLIKELY( !*pargc || ( strcmp( (*pargv)[ 0 ], "promote" ) && strcmp( (*pargv)[ 0 ], "status" ) ) ) ) { + FD_LOG_ERR(( "usage: firedancer failover [OPTIONS]" )); + } + int status = !strcmp( (*pargv)[ 0 ], "status" ); + (*pargc)--; + (*pargv)++; + failover_cmd_args( pargc, pargv, args, status ); +} + +/* The error string when a keypair other than the identity this machine runs is given */ +static char const * +holds_target_name( char const * target_b58 ) { + static char buf[ 192 ]; + FD_TEST( fd_cstr_printf_check( buf, sizeof(buf), NULL, "this machine runs `%s`, hand it off first, or pass --force to stop managing it here", + target_b58 ? target_b58 : "the target identity" ) ); + return buf; +} + + +/* The installed identity relative to the target and the junk key. */ +static char const * +installed_name( fd_adminctl_failover_status_resp_t const * resp ) { + if( !memcmp( resp->installed, resp->target, 32UL ) ) return "the target"; + if( !memcmp( resp->installed, resp->junk, 32UL ) ) return "this machine's junk key"; + return "another key"; +} + +/* Prints the identity lines of a status */ +static void +failover_identities_print( fd_adminctl_failover_status_resp_t const * resp ) { + FD_BASE58_ENCODE_32_BYTES( resp->target, target_b58 ); + FD_BASE58_ENCODE_32_BYTES( resp->installed, installed_b58 ); + FD_LOG_STDOUT(( "%-26s %s\n", "target identity:", target_b58 )); + FD_LOG_STDOUT(( "%-26s %s (%s)\n", "installed identity:", installed_b58, installed_name( resp ) )); + FD_LOG_STDOUT(( "%-26s %s\n", "runs target identity:", resp->holds ? "yes" : "no" )); +} + +static void +failover_floor_print( fd_adminctl_failover_status_resp_t const * resp ) { + if( FD_UNLIKELY( resp->promote_floor==FD_FAILOVER_SLOT_NULL ) ) FD_LOG_STDOUT(( "%-26s none known\n", "coverage floor:" )); + else FD_LOG_STDOUT(( "%-26s slot %lu\n", "coverage floor:", resp->promote_floor )); +} + +static void +failover_confirm( args_t const * args, + fd_adminctl_failover_status_resp_t const * status ) { + if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_PROMOTE ) { + if( FD_LIKELY( status ) ) { + failover_identities_print( status ); + FD_LOG_STDOUT(( "%-26s %s\n", "other holder:", other_holder( status->promote_result ) )); + failover_floor_print( status ); + FD_LOG_STDOUT(( "force only at most one machine at a time.\n" )); + } + FD_LOG_STDOUT(( "This takes the identity upon your command. No other machine may run it or be in a\n" + "promotion, including any validator running this identity without failover.\n" + "Run `failover status` on the other machine. It must show `runs target identity: no`\n" + "with no promotion in progress, or that machine must be stopped or otherwise\n" + "unable to vote. Never promote both machines at once.\n" + "WARNING: --force skips every check on the other machine. Use it only when the\n" + "other machine is not signing votes, or both machines may double-vote.\n" + "Promotion tries eligible saved vote history first, then starts from an EMPTY\n" + "history with a bound covering replay and known votes. Missing votes and leader\n" + "windows above that bound remain unprotected.\n" )); + } + if( FD_UNLIKELY( !args->failover.yes && isatty( STDIN_FILENO ) ) ) { + if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) { + FD_LOG_STDOUT(( "This asks the remote validator running this identity to hand it to this machine. It keeps\n" + "the identity if this machine is more than 64 slots behind its finality anchor.\n" + "Otherwise, it gives up the identity before this validator adopts its history.\n" + "If replay stalls, the handoff can fail with neither machine voting.\n" + "Check that this machine is caught up. Type yes to continue: " )); + } else { + FD_LOG_STDOUT(( "Type yes to continue: " )); + } + char line[ 16 ] = {0}; + if( FD_UNLIKELY( !fgets( line, sizeof(line), stdin ) ) ) FD_LOG_ERR(( "no confirmation given" )); + if( FD_UNLIKELY( strcmp( line, "yes\n" ) ) ) FD_LOG_ERR(( "not confirmed, nothing was done" )); + } +} + +/* Prints the response to `failover status`. */ +static void +failover_status_print( fd_adminctl_failover_status_resp_t const * resp ) { + if( FD_UNLIKELY( !resp->enabled ) ) { + FD_LOG_STDOUT(( "%-26s %s\n", "failover:", "disabled" )); + return; + } + + FD_LOG_STDOUT(( "%-26s %s\n", "failover:", "enabled" )); + failover_identities_print( resp ); + FD_BASE58_ENCODE_32_BYTES( resp->junk, junk_b58 ); + FD_LOG_STDOUT(( "%-26s %s\n", "junk key:", junk_b58 )); + if( FD_UNLIKELY( resp->request_paused ) ) { + FD_LOG_STDOUT(( "%-26s %s, paused at its deadline, `failover promote` resumes it\n", "action:", action_name( resp->action ) )); + } else { + FD_LOG_STDOUT(( "%-26s %s\n", "action:", action_name( resp->action ) )); + } + FD_LOG_STDOUT(( "%-26s %s\n", "stuck:", !resp->stuck ? "no" : resp->request_paused ? "yes, the handoff request is paused, see the log" : "yes, see the log" )); + FD_LOG_STDOUT(( "%-26s %s\n", "link:", resp->link_statelink_state ] : "unknown" )); + FD_LOG_STDOUT(( "%-26s %s\n", "peer runs target identity:", !resp->peer_holds_valid ? "unknown, no authenticated failover session" : resp->peer_holds ? "yes" : "no" )); + if( FD_UNLIKELY( !resp->peer_boot_id ) ) FD_LOG_STDOUT(( "%-26s none paired since boot\n", "peer boot id:" )); + else FD_LOG_STDOUT(( "%-26s %016lx\n", "peer boot id:", resp->peer_boot_id )); + if( FD_UNLIKELY( !resp->peer_addr && resp->holds ) ) { + FD_LOG_STDOUT(( "%-26s n/a, this machine runs the identity\n", "peer address:" )); + } else if( FD_UNLIKELY( !resp->peer_addr ) ) { + FD_LOG_STDOUT(( "%-26s unknown, gossip has no contact info for the identity from another host\n", "peer address:" )); + } else { + FD_LOG_STDOUT(( "%-26s " FD_IP4_ADDR_FMT ":%u, from %s\n", "peer address:", FD_IP4_ADDR_FMT_ARGS( resp->peer_addr ), (uint)resp->peer_port, + resp->peer_addr_cmd ? "--address" : "gossip" )); + } + + if( FD_UNLIKELY( !resp->handoff_id ) ) { + FD_LOG_STDOUT(( "%-26s none since boot\n", "last handoff:" )); + } else { + FD_LOG_STDOUT(( "%-26s %lu, %s\n", "last handoff:", resp->handoff_id, + resp->handoff_resulthandoff_result ] : "unknown" )); + } + + FD_LOG_STDOUT(( "%-26s %s\n", "other holder:", other_holder( resp->promote_result ) )); + FD_LOG_STDOUT(( "%-26s %s\n", "takeover adopts:", + resp->promote_sourcepromote_source ] : "unknown" )); + failover_floor_print( resp ); +} + +/* Sends req to the failover tile via adminctl and waits for the response. Returns the adminctl result. */ +static ulong FD_FN_SENSITIVE +failover_request( fd_adminctl_t * adminctl, + fd_adminctl_failover_req_t const * req, + char const * keypair_path, + void * resp, + ulong resp_max, + ulong * resp_sz ) { + void * payload = NULL; + ulong payload_max = 0UL; + ulong slot_idx = fd_adminctl_reserve( adminctl, &payload, &payload_max ); + if( FD_UNLIKELY( slot_idx==ULONG_MAX ) ) FD_LOG_ERR(( "all admin command slots are busy" )); + ulong payload_sz = keypair_path ? sizeof(fd_adminctl_failover_prepare_t) : sizeof(*req); + if( FD_UNLIKELY( payload_sz>payload_max ) ) FD_LOG_ERR(( "adminctl failover payload too large" )); + fd_memcpy( payload, req, sizeof(*req) ); + if( keypair_path ) { + uchar const * keypair = fd_keyload_load( keypair_path, 0 ); + fd_memcpy( ((fd_adminctl_failover_prepare_t *)payload)->keypair, keypair, 64UL ); + fd_keyload_unload( keypair, 0 ); + } + ulong cmd = req->cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? FD_ADMINCTL_CMD_FAILOVER_STATUS : FD_ADMINCTL_CMD_FAILOVER_PROMOTE; + fd_adminctl_publish( adminctl, slot_idx, cmd, payload_sz ); + return fd_adminctl_wait_response( adminctl, slot_idx, resp, resp_max, resp_sz ); +} + +static void __attribute__((noreturn)) +fail_disabled( int status ) { + if( status ) FD_LOG_ERR(( "`failover status` is unavailable because this validator has no failover command bus" )); + FD_LOG_ERR(( "failover is not running on this validator, it needs [paths.vote_account] and [failover.enabled] true, then a restart" )); +} + +static void __attribute__((noreturn)) +fail_incompatible( void ) { + FD_LOG_ERR(( "running validator returned an incompatible failover response, use the firedancer binary the validator runs" )); +} + +/* failover_status_fetch reads the live `failover status` into status. + Returns 1 when it is valid, 0 when the validator gave no usable + answer. */ +static int +failover_status_fetch( fd_adminctl_t * adminctl, + fd_adminctl_failover_status_resp_t * status ) { + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_STATUS }; + ulong status_sz = 0UL; + ulong result = failover_request( adminctl, &req, NULL, status, sizeof(*status), &status_sz ); + if( FD_UNLIKELY( result==FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ) ) fail_disabled( 0 ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS || status_sz!=sizeof(*status) || + status->version!=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION ) ) return 0; + if( FD_UNLIKELY( !status->enabled ) ) fail_disabled( 0 ); + return 1; +} + +/* Run sanity checks before invoking failover. */ +static void FD_FN_SENSITIVE +failover_precheck( fd_adminctl_failover_status_resp_t const * status, + args_t const * args ) { + char const * cmd_name = fd_adminctl_failover_cmd_typed( (ulong)args->failover.cmd ); + if( args->failover.keypair_path[ 0 ] ) { + uchar pubkey[ 32 ]; + fd_sha512_t sha512[ 1 ]; + uchar const * keypair = fd_keyload_load( args->failover.keypair_path, 0 ); + FD_TEST( fd_sha512_join( fd_sha512_new( sha512 ) ) ); + fd_ed25519_public_from_private( pubkey, keypair, sha512 ); + fd_keyload_unload( keypair, 0 ); + int same = !memcmp( pubkey, status->target, 32UL ); + if( FD_UNLIKELY( same && status->holds ) ) { + FD_LOG_ERR(( "`%s` refused, %s", cmd_name, control_result_name( FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER ) )); + } + if( FD_UNLIKELY( !same && status->holds && status->promote_floor!=FD_FAILOVER_SLOT_NULL && !args->failover.force ) ) { + FD_BASE58_ENCODE_32_BYTES( status->target, target_b58 ); + FD_LOG_ERR(( "`%s` refused, %s", cmd_name, holds_target_name( target_b58 ) )); + } + if( FD_UNLIKELY( !same && status->action!=FD_FAILOVER_ACTION_IDLE && !args->failover.force ) ) { + FD_LOG_ERR(( "`%s` refused, a transition is running, wait for `failover status` to show idle or pass --force", cmd_name )); + } + return; + } + if( FD_UNLIKELY( status->holds && + !( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF && + status->request_paused && status->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) ) { + FD_LOG_ERR(( "`%s` refused, %s", cmd_name, control_result_name( FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER ) )); + } +} + +/* Send the command to the failover tile and print the result. */ +static void +failover_cmd_fn( args_t * args, + config_t * config ) { + fd_adminctl_t * adminctl = adminctl_client_attach( config, args->failover.name ); + /* run prechecks on the current status */ + int status = args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_STATUS; + if( !status ) { + fd_adminctl_failover_status_resp_t live; + int live_ok = failover_status_fetch( adminctl, &live ); + if( live_ok && !args->failover.yes && isatty( STDIN_FILENO ) ) failover_precheck( &live, args ); + failover_confirm( args, live_ok ? &live : NULL ); + } + + /* fetch args */ + fd_adminctl_failover_req_t req; + fd_memset( &req, 0, sizeof(req) ); + req.version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION; + req.cmd = (ulong)args->failover.cmd; + if( FD_UNLIKELY( args->failover.yes ) ) req.flags |= FD_ADMINCTL_FAILOVER_FLAG_YES; + if( FD_UNLIKELY( args->failover.force ) ) req.flags |= FD_ADMINCTL_FAILOVER_FLAG_FORCE; + req.addr = args->failover.addr; + req.port = args->failover.port; + + union { + fd_adminctl_failover_control_resp_t control; + fd_adminctl_failover_status_resp_t status; + } resp = { 0 }; + ulong resp_sz = 0UL; + char const * keypair = args->failover.keypair_path[ 0 ] ? args->failover.keypair_path : NULL; + + /* run the command */ + ulong result = failover_request( adminctl, &req, keypair, &resp, sizeof(resp), &resp_sz ); + char const * cmd_name = fd_adminctl_failover_cmd_typed( (ulong)args->failover.cmd ); + char const * refusal = control_result_name( result ); + + if( FD_UNLIKELY( refusal ) ) FD_LOG_ERR(( "`%s` refused, %s", cmd_name, refusal )); + + /* show the result */ + switch( result ) { + case FD_ADMINCTL_RESULT_SUCCESS: + if( FD_UNLIKELY( resp_sz!=(status ? sizeof(resp.status) : sizeof(resp.control)) || + (status ? resp.status.version : resp.control.version)!=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION || + (status && resp.status.mode!=FD_FAILOVER_MODE_ALPENGLOW) ) ) { + fail_incompatible(); + } + if( status ) { + failover_status_print( &resp.status ); + } else { + ulong id = resp.control.handoff_id; + FD_LOG_STDOUT(( "%-26s accepted\n", cmd_name )); + FD_LOG_STDOUT(( "%-26s %s\n", "runs target identity:", resp.control.holds ? "yes" : "no" )); + FD_LOG_STDOUT(( "%-26s %s\n", "action:", action_name( resp.control.action ) )); + if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) { + FD_LOG_STDOUT(( "%-26s %lu, done when `failover status` here shows runs target identity: yes and last handoff: %lu, taken\n", "handoff:", id, id )); + } else { + FD_LOG_STDOUT(( "%-26s done when `failover status` here shows runs target identity: yes and action: idle\n", "promotion:" )); + } + } + break; + case FD_FAILOVER_CONTROL_RESULT_BUSY: + FD_LOG_ERR(( "another failover command is in flight, try again" )); + case FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE: + if( status ) FD_LOG_ERR(( "the validator's failover tile did not answer in time" )); + FD_LOG_ERR(( "the validator's failover tile did not answer in time, the outcome is unknown, check `failover status`" )); + case FD_ADMINCTL_RESULT_UNKNOWN_COMMAND: + if( status ) { + FD_LOG_STDOUT(( "%-26s %s\n", "failover:", "disabled" )); + break; + } + fail_disabled( 0 ); + case FD_ADMINCTL_RESULT_UNSUPPORTED: + fail_disabled( status ); + case FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH: + case FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH: + FD_LOG_ERR(( "failover is incompatible with the running validator, use the same firedancer version this validator runs" )); + default: + FD_LOG_ERR(( "unexpected failover result %lu", result )); + } +} + +static void +failover_status_args_help( fd_action_help_t * help ) { + fd_action_help_arg( help, "--name", "", "Name of the validator instance to attach to, if more than one is\n" + "running on this host" ); +} + +static void +failover_args_help( fd_action_help_t * help ) { + fd_action_help_arg( help, "", NULL, "Request failover takeover on this validator (promote), or read its failover state (status)." ); + fd_action_help_arg( help, "[]", NULL, "Path to the keypair of the identity to take." ); + failover_status_args_help( help ); + fd_action_help_arg( help, "--yes", NULL, "Bypass the interactive confirmation prompt." ); + fd_action_help_arg( help, "--force", NULL, "Forcibly take the identity without asking the machine running it, skipping every\n" + "check. Only use when the other machine cannot vote.\n" ); + fd_action_help_arg( help, "--address", "
", "Only without --force. The target machine's IPv4 address or host name" ); + fd_action_help_arg( help, "--port", "", "Only without --force. The target machine's failover port, 8010 by default" ); +} + +static void +failover_promote_perm( args_t * args, + fd_cap_chk_t * chk, + config_t const * config FD_PARAM_UNUSED ) { + if( args->failover.keypair_path[ 0 ] ) { + fd_cap_chk_raise_rlimit( chk, "failover promote", RLIMIT_MEMLOCK, 5UL*FD_SHMEM_NORMAL_PAGE_SZ, + "lock the target keypair in memory" ); + } +} + +action_t fd_action_failover = { + .name = "failover", + .args = failover_args, + .fn = failover_cmd_fn, + .require_config = 0, + .perm = failover_promote_perm, + .description = "Take the identity that another validator runs or read the failover status", + .detail = "Hands off an identity between validators. Each boots under its own\n" + "[paths.identity_key], and all validators must use the same [paths.vote_account].\n" + "On the machine that should take over, pass the keypair of the target identity.\n" + "A validator with [failover.boot_to_standby] boots as a standby under a junk key\n" + "generated in memory and loads the keypair in its [paths.identity_key] as the\n" + "identity to take over, so `failover promote` there reads the keypair from the\n" + "configuration instead of the command line.\n" + "\n" + "Asks the machine running the identity to hand it over. With --force,\n" + "takes the identity without handover.\n" + "This command asks for confirmation unless --yes is given.\n" + "Use `failover status` to check handover progress.\n" + "\n" + "This command does not start a validator. It attaches to one that is already\n" + "running. If multiple validators are running, pass --name to select one.\n", + .usage = "failover promote [] [--name ] [--yes] [--force] [--address
] [--port ]\n" + " firedancer failover status [--name ]", + .args_help = failover_args_help, +}; diff --git a/src/app/firedancer/commands/set_identity.c b/src/app/firedancer/commands/set_identity.c index 6887dd660fb..4ec132aea9b 100644 --- a/src/app/firedancer/commands/set_identity.c +++ b/src/app/firedancer/commands/set_identity.c @@ -132,10 +132,13 @@ set_identity( args_t * args, case FD_SET_IDENTITY_RESULT_INVALID_VOTE_HISTORY: FD_LOG_ERR(( "Failed to set identity: the vote history file is invalid or does not belong to the new identity. " "If you believe the vote history file is valid, please contact the Firedancer team for additional assistance." )); + case FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH: + FD_LOG_ERR(( "Failed to set identity: the public key in the keypair does not match the public key derived " + "from its private key." )); case FD_ADMINCTL_RESULT_UNKNOWN_COMMAND: + case FD_ADMINCTL_RESULT_UNSUPPORTED: case FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH: case FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH: - case FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH: FD_LOG_ERR(( "Failed to set identity: the command was not able to successfully communicate " "with the running Firedancer process. It is possible that you are running the " "command from an older or newer version of Firedancer that is no longer compatible." )); diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index 32c2bac2711..0a2157e10fe 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -103,6 +103,13 @@ telemetry = true # if one is not provided (or has not already been generated) and # place it in this path. If no path is provided, it is defaulted # to inside the base directory, under path `identity.json`. + # + # The validator boots under this identity. For failover, a standby + # machine that waits to take over another validator's identity + # should boot under an unstaked randomized key, created using + # `firedancer keys new`, or set [failover.boot_to_standby] to boot + # under an ephemeral junk key generated in memory, with the keypair + # here as the identity it can take over via hand-off. identity_key = "" # Absolute path to a `keypair.json` containing the voting account @@ -1974,6 +1981,52 @@ telemetry = true # loading. delay_startup = true +# Firedancer validators support a coordinated failover protocol. A +# handoff transfers the final vote history after the active stops signing. +# `failover promote --force` bypasses peer checks and can discard earlier +# lockouts or allow both validators to vote. +# +# The protocol works by hosting a TCP listener on your validator. Any +# peer in possession of the currently active identity private key can +# demote this validator to the junk identity by sending a specially +# signed packet proving that possession. +# +# Between two Firedancer validators, only hand off the identity with +# `failover promote`. Do not use `set-identity --vote-history-file` +# for this. The failover protocol passes the final vote history +# directly and checks it against what this validator replayed. Vote +# history files are only for handing off an identity from a validator that +# does not run failover, such as Agave. +[failover] + # Whether to enable failover. It only takes effect with + # [paths.vote_account] set. The validator boots under + # [paths.identity_key], and when `failover promote ` is + # run on a standby host, it takes over the identity in that keypair + # from the machine that is actively running it. + enabled = true + + # Whether to boot as a standby under a junk identity generated in + # memory at each start, instead of under [paths.identity_key]. The + # junk private key is never written out to a file. The keypair in + # [paths.identity_key] is then the identity this machine takes with + # `failover promote`. Set it on a standby machine that should take + # over a staked identity another machine runs. This prevents a + # restarted standby from double-voting under the same staked identity + # while the other active machine is already voting with it. + # Requires [failover.enabled] and [paths.vote_account]. + boot_to_standby = false + + # The address to listen on. If enabled, by default the listener is + # exposed to the network. This is safe as possession of the private + # key is required to take over the identity, but could be a vector + # for a targeted denial of service. For added reliability, restrict + # the listener to a private network. + listen_address = "0.0.0.0" + + # The port to listen on for handoff requests. The other validator + # dials 8010 unless `failover promote --port` gives another port. + listen_port = 8010 + # These options can be useful for development, but should not be used # when connecting to a live cluster, as they may cause the validator to # be unstable or have degraded performance or security. The program diff --git a/src/app/firedancer/main.c b/src/app/firedancer/main.c index d99ad4482b5..194cc09a21b 100644 --- a/src/app/firedancer/main.c +++ b/src/app/firedancer/main.c @@ -106,6 +106,7 @@ extern fd_topo_run_tile_t fd_tile_bundle; extern fd_topo_run_tile_t fd_tile_genesi; extern fd_topo_run_tile_t fd_tile_ipecho; extern fd_topo_run_tile_t fd_tile_admin; +extern fd_topo_run_tile_t fd_tile_failov; extern fd_topo_run_tile_t fd_tile_gossvf; extern fd_topo_run_tile_t fd_tile_gossip; @@ -163,6 +164,7 @@ fd_topo_run_tile_t * TILES[] = { &fd_tile_genesi, &fd_tile_ipecho, &fd_tile_admin, + &fd_tile_failov, &fd_tile_solcap, &fd_tile_snapmk, &fd_tile_snapzp, @@ -188,6 +190,7 @@ extern action_t fd_action_add_authorized_voter; extern action_t fd_action_remove_all_authorized_voters; extern action_t fd_action_set_identity; extern action_t fd_action_get_identity; +extern action_t fd_action_failover; extern action_t fd_action_ps; extern action_t fd_action_monitor_gossip; extern action_t fd_action_snapshot_create; @@ -212,6 +215,7 @@ action_t * ACTIONS[] = { &fd_action_monitor_gossip, &fd_action_set_identity, &fd_action_get_identity, + &fd_action_failover, &fd_action_ps, &fd_action_snapshot_create, &fd_action_wait, diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index 1ddd15f83ec..6dd2d28fd25 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -1,4 +1,5 @@ #include "topology.h" +#include "../../disco/keyguard/fd_keyload.h" #include "../shared/fd_config_json.h" #include "../../discof/poh/fd_poh.h" @@ -11,6 +12,8 @@ #include "../../discof/votor/fd_votor_tile.h" #include "../../disco/keyguard/fd_keyguard.h" #include "../../ballet/bls/fd_bls.h" +#include "../../discof/failover/fd_failover_bus.h" +#include "../../discof/failover/fd_failover_proto.h" #include "../../discof/backup/fd_snapmk_tile.h" #include "../../discof/backup/fd_snapsv_tile.h" #include "../../disco/shred/fd_shred_tile.h" @@ -283,6 +286,7 @@ fd_topo_initialize( config_t * config ) { int leader_enabled = !!config->firedancer.layout.enable_block_production; int rserve_enabled = config->tiles.rserve.enabled; int efficient_mode = !strcmp( config->firedancer.layout.mode, "efficient" ); + int failover_enabled = config->firedancer.failover.enabled && config->paths.vote_account[ 0 ]; if( FD_UNLIKELY( snapmk_enabled ) ) { FD_CHECK_ERR( config->firedancer.snapshots.max_full_snapshots_to_keep, @@ -315,6 +319,9 @@ fd_topo_initialize( config_t * config ) { fd_topob_wksp( topo, "votor" ); fd_topob_wksp( topo, "sign" )->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_NEVER; fd_topob_wksp( topo, "admin" )->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_NEVER; + /* The failov tile keeps TLS session keys in its scratch, so its + workspace stays out of core dumps like admin and sign. */ + if( FD_LIKELY( failover_enabled ) ) fd_topob_wksp( topo, "failov" )->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_NEVER; if( leader_enabled ) { fd_topob_wksp( topo, "quic" ); @@ -402,8 +409,20 @@ fd_topo_initialize( config_t * config ) { fd_topob_wksp( topo, "votor_sign" ); fd_topob_wksp( topo, "sign_votor" ); + if( FD_LIKELY( failover_enabled ) ) { + fd_topob_wksp( topo, "failov_sign" ); + fd_topob_wksp( topo, "sign_failov" ); + } + fd_topob_wksp( topo, "execrp_replay" ); fd_topob_wksp( topo, "admin_replay" ); + if( FD_LIKELY( failover_enabled ) ) { + fd_topob_wksp( topo, "admin_failov" ); + fd_topob_wksp( topo, "failov_admin" ); + fd_topob_wksp( topo, "failov_votor" ); + fd_topob_wksp( topo, "votor_hist" ); + if( !config->tiles.gui.enabled ) fd_topob_wksp( topo, "gossip_ciseen" ); + } if( FD_LIKELY( snapshots_enabled ) ) { fd_topob_wksp( topo, "snapct" ); @@ -506,6 +525,16 @@ fd_topo_initialize( config_t * config ) { FOR(execrp_tile_cnt) fd_topob_link( topo, "replay_execrp", "replay_execrp", 256UL, sizeof(fd_execrp_task_msg_t), 1UL ); /**/ fd_topob_link( topo, "admin_replay", "admin_replay", 32UL, 0UL, 1UL ); /**/ fd_topob_link( topo, "replay_admin", "admin_replay", 32UL, 0UL, 1UL ); + if( FD_LIKELY( failover_enabled ) ) { + /**/ fd_topob_link( topo, "admin_failov", "admin_failov", 32UL, FD_FAILOVER_BUS_MTU, 1UL ); + /**/ fd_topob_link( topo, "failov_admin", "failov_admin", 32UL, FD_FAILOVER_BUS_MTU, 1UL ); + /**/ fd_topob_link( topo, "failov_votor", "failov_votor", 32UL, FD_VOTOR_ADOPT_REQ_HDR_SZ+FD_FAILOVER_STATE_MAX, 1UL ); + /**/ fd_topob_link( topo, "votor_failov", "failov_votor", 32UL, sizeof(fd_votor_adopt_result_t), 1UL ); + /**/ fd_topob_link( topo, "votor_hist", "votor_hist", 1024UL, sizeof(fd_votor_hist_msg_t), 1UL ); + if( !config->tiles.gui.enabled ) { + fd_topob_link( topo, "gossip_ciseen", "gossip_ciseen", 65536UL*2UL, sizeof(fd_gossip_update_message_t), 1UL ); + } + } if( leader_enabled ) { /**/ fd_topob_link( topo, "dedup_resolv", "dedup_resolv", 16384UL, FD_TPU_PARSED_MTU, 1UL ); FOR(resolv_tile_cnt) fd_topob_link( topo, "resolv_pack", "resolv_pack", 4096UL, FD_TPU_RESOLVED_MTU, 1UL ); @@ -530,6 +559,11 @@ fd_topo_initialize( config_t * config ) { FOR(sign_tile_cnt-1) fd_topob_link( topo, "repair_sign", "repair_sign", 256UL, FD_REPAIR_SIG_SER_MAX, 1UL ); /* See repair_tile.c for explanation */ FOR(sign_tile_cnt-1) fd_topob_link( topo, "sign_repair", "sign_repair", 256UL, sizeof(fd_ed25519_sig_t), 1UL ); + if( FD_LIKELY( failover_enabled ) ) { + /**/ fd_topob_link( topo, "failov_sign", "failov_sign", 128UL, 130UL, 1UL ); + /**/ fd_topob_link( topo, "sign_failov", "sign_failov", 128UL, sizeof(fd_ed25519_sig_t), 1UL ); + } + FOR(shred_tile_cnt) fd_topob_link( topo, "shred_out", "shred_out", shred_depth, sizeof(fd_shred_message_t), FD_SHRED_STEM_BURST ); /**/ fd_topob_link( topo, "repair_out", "repair_out", repair_out_depth, sizeof(fd_repair_replay_fec_t), 1UL ); @@ -744,6 +778,13 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_tile_out( topo, "replay", 0UL, "replay_admin", 0UL ); /**/ fd_topob_tile_out( topo, "admin", 0UL, "admin_replay", 0UL ); /**/ fd_topob_tile_in ( topo, "admin", 0UL, "metric_in", "replay_admin", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + if( FD_LIKELY( failover_enabled ) ) { + if( !config->tiles.gui.enabled ) { + fd_topob_tile_out( topo, "gossip", 0UL, "gossip_ciseen", 0UL ); + } + /**/ fd_topob_tile_out( topo, "admin", 0UL, "admin_failov", 0UL ); + /**/ fd_topob_tile_in ( topo, "admin", 0UL, "metric_in", "failov_admin", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + } FOR(execrp_tile_cnt) fd_topob_tile_in ( topo, "execrp", i, "metric_in", "replay_execrp", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); FOR(execrp_tile_cnt) fd_topob_tile_out( topo, "execrp", i, "execrp_replay", i ); @@ -892,6 +933,11 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_rserve", 0UL ); } + if( FD_LIKELY( failover_enabled ) ) { + /**/ fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "failov_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_failov", 0UL ); + } + if( FD_UNLIKELY( rpc_enabled ) ) { fd_topob_link( topo, "rpc_replay", "rpc_replay", 8UL, 0UL, 1UL ); fd_topob_tile_out( topo, "rpc", 0UL, "rpc_replay", 0UL ); @@ -1043,6 +1089,18 @@ fd_topo_initialize( config_t * config ) { } } + if( FD_LIKELY( failover_enabled ) ) { + /**/ fd_topob_tile( topo, "failov", "failov", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 ); + + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "gossip_ciseen", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "admin_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_admin", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_hist", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_votor", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_sign", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "sign_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED ); + } if( FD_LIKELY( !is_auto_affinity ) ) { if( FD_UNLIKELY( affinity_tile_cnttile_cnt ) ) FD_LOG_ERR(( "The topology you are using has %lu tiles, but the CPU affinity specified in the config tile as [layout.affinity] only provides for %lu cores. " @@ -1102,6 +1160,11 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_tile_in ( topo, "rpc", 0UL, "metric_in", "votor_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); } + if( FD_LIKELY( failover_enabled ) ) { + /**/ fd_topob_tile_in ( topo, "votor", 0UL, "metric_in", "failov_votor", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "votor", 0UL, "votor_hist", 0UL ); + /**/ fd_topob_tile_out( topo, "votor", 0UL, "votor_failov", 0UL ); + } /**/ fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "votor_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "votor", 0UL, "votor_sign", 0UL ); /**/ fd_topob_tile_in ( topo, "votor", 0UL, "metric_in", "sign_votor", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED ); @@ -1368,6 +1431,13 @@ wfs_decode_bank_hash( fd_config_t const * config, void fd_topo_configure_tile( fd_topo_tile_t * tile, fd_config_t * config ) { + /* With [failover.boot_to_standby] every tile boots under the junk key + generated at boot, and [paths.identity_key] is only the handoff + target. The sign and failov tiles keep the configured path, they + load the target from it. */ + int boot_to_standby = config->firedancer.failover.boot_to_standby && fd_topo_find_tile( &config->topo, "failov", 0UL )!=ULONG_MAX; + char const * identity_key_path = boot_to_standby ? FD_KEYLOAD_FAILOVER_JUNK : config->paths.identity_key; + if( FD_UNLIKELY( !strcmp( tile->name, "metric" ) ) ) { if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->tiles.metric.prometheus_listen_address, &tile->metric.prometheus_listen_addr ) ) ) @@ -1376,7 +1446,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "event" ) ) ) { - fd_cstr_ncpy( tile->event.identity_key_path, config->paths.identity_key, sizeof(tile->event.identity_key_path) ); + fd_cstr_ncpy( tile->event.identity_key_path, identity_key_path, sizeof(tile->event.identity_key_path) ); fd_cstr_ncpy( tile->event.url, config->tiles.event.url, sizeof(tile->event.url) ); fd_cstr_ncpy( tile->event.action, config->action, sizeof(tile->event.action) ); FD_TEST( fd_cstr_printf_check( tile->event.accounts_path, sizeof(tile->event.accounts_path), NULL, "%s", config->paths.accounts ) ); @@ -1439,11 +1509,23 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "admin" ) ) ) { - fd_cstr_ncpy( tile->admin.identity_key_path, config->paths.identity_key, sizeof(tile->admin.identity_key_path) ); + fd_cstr_ncpy( tile->admin.identity_key_path, identity_key_path, sizeof(tile->admin.identity_key_path) ); + + } else if( FD_UNLIKELY( !strcmp( tile->name, "failov" ) ) ) { + + fd_cstr_ncpy( tile->failov.identity_key_path, config->paths.identity_key, sizeof(tile->failov.identity_key_path) ); + fd_cstr_ncpy( tile->failov.vote_account_path, config->paths.vote_account, sizeof(tile->failov.vote_account_path) ); + tile->failov.boot_to_standby = boot_to_standby; + if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->firedancer.failover.listen_address, &tile->failov.listen_addr ) ) ) + FD_LOG_ERR(( "[failover.listen_address] `%s` must be an IPv4 address, host names are not accepted", config->firedancer.failover.listen_address )); + tile->failov.port = config->firedancer.failover.listen_port; + fd_cstr_ncpy( tile->failov.gossip_host, config->firedancer.gossip.host, sizeof(tile->failov.gossip_host) ); + tile->failov.gossip_addr.addr = config->net.ip_addr; + tile->failov.gossip_addr.port = fd_ushort_bswap( config->gossip.port ); } else if( FD_UNLIKELY( !strcmp( tile->name, "gossvf") ) ) { - fd_cstr_ncpy( tile->gossvf.identity_key_path, config->paths.identity_key, sizeof(tile->gossvf.identity_key_path) ); + fd_cstr_ncpy( tile->gossvf.identity_key_path, identity_key_path, sizeof(tile->gossvf.identity_key_path) ); tile->gossvf.tcache_depth = 1<<22UL; /* TODO: user defined option */ tile->gossvf.shred_version = 0U; tile->gossvf.allow_private_address = config->development.gossip.allow_private_address; @@ -1467,7 +1549,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->gossip.net_ip_addr = config->net.ip_addr; tile->gossip.ip_addr = config->net.ip_addr; tile->gossip.bind_ip_addr = config->net.ip_addr; - fd_cstr_ncpy( tile->gossip.identity_key_path, config->paths.identity_key, sizeof(tile->gossip.identity_key_path) ); + fd_cstr_ncpy( tile->gossip.identity_key_path, identity_key_path, sizeof(tile->gossip.identity_key_path) ); tile->gossip.expected_shred_version = config->consensus.expected_shred_version; tile->gossip.max_entries = config->tiles.gossip.max_entries; tile->gossip.boot_timestamp_nanos = config->boot_timestamp_nanos; @@ -1562,13 +1644,13 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } } tile->rotor.repair_sign_cnt = config->firedancer.layout.sign_tile_count - 1; /* -1 because this excludes the keyguard client */ - fd_cstr_ncpy( tile->rotor.identity_key_path, config->paths.identity_key, sizeof(tile->rotor.identity_key_path) ); + fd_cstr_ncpy( tile->rotor.identity_key_path, identity_key_path, sizeof(tile->rotor.identity_key_path) ); } else if( FD_UNLIKELY( !strcmp( tile->name, "rserve" ) ) ) { tile->rserve.repair_serve_listen_port = config->tiles.rserve.repair_serve_listen_port; tile->rserve.max_shreds_per_block = config->limits.max_shreds_per_block; tile->rserve.ping_cache_entries = 1UL<<16; /* TODO: Configure this from some global metric? */ tile->rserve.blockdb_max = 1UL<<15; /* ~20.6 KiB each. TODO: make configurable */ - fd_cstr_ncpy( tile->rserve.identity_key_path, config->paths.identity_key, sizeof(tile->rserve.identity_key_path) ); + fd_cstr_ncpy( tile->rserve.identity_key_path, identity_key_path, sizeof(tile->rserve.identity_key_path) ); } else if( FD_UNLIKELY( !strcmp( tile->name, "replay" ) )) { @@ -1584,7 +1666,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->replay.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX ); FD_TEST( tile->replay.txncache_obj_id !=ULONG_MAX ); - fd_cstr_ncpy( tile->replay.identity_key_path, config->paths.identity_key, sizeof(tile->replay.identity_key_path) ); + fd_cstr_ncpy( tile->replay.identity_key_path, identity_key_path, sizeof(tile->replay.identity_key_path) ); tile->replay.ip_addr = config->net.ip_addr; fd_cstr_ncpy( tile->replay.vote_account_path, config->paths.vote_account, sizeof(tile->replay.vote_account_path) ); @@ -1649,7 +1731,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->votor.quic_server_listen_port = config->firedancer.development.votor.quic_server_listen_port; tile->votor.ip_addr = config->net.ip_addr; tile->votor.max_live_slots = config->firedancer.runtime.max_live_slots; - fd_cstr_ncpy( tile->votor.identity_key_path, config->paths.identity_key, sizeof(tile->votor.identity_key_path) ); + fd_cstr_ncpy( tile->votor.identity_key_path, identity_key_path, sizeof(tile->votor.identity_key_path) ); fd_cstr_ncpy( tile->votor.vote_history_path, config->paths.vote_history, sizeof(tile->votor.vote_history_path) ); tile->votor.target_gid = config->gid; tile->votor.target_uid = config->uid; @@ -1734,7 +1816,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, PARSE_BUNDLE_PUBKEY( pack, tip_payment_program_addr ); PARSE_BUNDLE_PUBKEY( pack, tip_distribution_authority ); tile->pack.bundle.commission_bps = config->tiles.bundle.commission_bps; - fd_cstr_ncpy( tile->pack.bundle.identity_key_path, config->paths.identity_key, sizeof(tile->pack.bundle.identity_key_path) ); + fd_cstr_ncpy( tile->pack.bundle.identity_key_path, identity_key_path, sizeof(tile->pack.bundle.identity_key_path) ); fd_cstr_ncpy( tile->pack.bundle.vote_account_path, config->paths.vote_account, sizeof(tile->pack.bundle.vote_account_path) ); } else { fd_memset( &tile->pack.bundle, '\0', sizeof(tile->pack.bundle) ); @@ -1749,14 +1831,14 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->execle.report_runtime_diffs = config->development.event.report_runtime_diffs; } else if( FD_UNLIKELY( !strcmp( tile->name, "motor" ) ) ) { - fd_cstr_ncpy( tile->motor.identity_key_path, config->paths.identity_key, sizeof(tile->motor.identity_key_path) ); + fd_cstr_ncpy( tile->motor.identity_key_path, identity_key_path, sizeof(tile->motor.identity_key_path) ); tile->motor.execle_cnt = config->firedancer.layout.execle_tile_count; tile->motor.max_txn_per_slot = config->limits.max_txn_per_slot; } else if( FD_UNLIKELY( !strcmp( tile->name, "shred" ) ) ) { - fd_cstr_ncpy( tile->shred.identity_key_path, config->paths.identity_key, sizeof(tile->shred.identity_key_path) ); + fd_cstr_ncpy( tile->shred.identity_key_path, identity_key_path, sizeof(tile->shred.identity_key_path) ); tile->shred.fec_exposure = FD_SHRED_FIREDANCER_FEC_EXPOSURE; tile->shred.fec_resolver_depth = config->tiles.shred.max_pending_shred_sets; @@ -1775,6 +1857,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) ) ) { fd_cstr_ncpy( tile->sign.identity_key_path, config->paths.identity_key, sizeof(tile->sign.identity_key_path) ); + tile->sign.failover_junk_fd = -1; + tile->sign.failover_enabled = fd_topo_find_tile( &config->topo, "failov", 0UL )!=ULONG_MAX; + tile->sign.failover_boot_to_standby = boot_to_standby; tile->sign.authorized_voter_paths_cnt = config->firedancer.paths.authorized_voter_paths_cnt; for( ulong i=0UL; isign.authorized_voter_paths_cnt; i++ ) { @@ -1811,7 +1896,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->gui.max_txn_per_slot = config->limits.max_txn_per_slot; tile->gui.is_voting = strcmp( config->paths.vote_account, "" ); fd_cstr_ncpy( tile->gui.cluster, config->cluster, sizeof(tile->gui.cluster) ); - fd_cstr_ncpy( tile->gui.identity_key_path, config->paths.identity_key, sizeof(tile->gui.identity_key_path) ); + fd_cstr_ncpy( tile->gui.identity_key_path, identity_key_path, sizeof(tile->gui.identity_key_path) ); fd_cstr_ncpy( tile->gui.vote_key_path, config->paths.vote_account, sizeof(tile->gui.vote_key_path) ); fd_cstr_ncpy( tile->gui.accounts_database_path, config->paths.accounts, sizeof(tile->gui.accounts_database_path) ); fd_cstr_ncpy( tile->gui.gui_database_path, config->paths.guidb, sizeof(tile->gui.gui_database_path) ); @@ -1846,7 +1931,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->rpc.accdb_epoch_fseq_obj_id = fd_pod_query_ulong( config->topo.props, "accdb_epoch.rpc", ULONG_MAX ); FD_TEST( tile->rpc.accdb_epoch_fseq_obj_id!=ULONG_MAX ); - fd_cstr_ncpy( tile->rpc.identity_key_path, config->paths.identity_key, sizeof(tile->rpc.identity_key_path) ); + fd_cstr_ncpy( tile->rpc.identity_key_path, identity_key_path, sizeof(tile->rpc.identity_key_path) ); tile->rpc.snapshot_server_enabled = fd_topo_find_tile( &config->topo, "snapsv", 0UL )!=ULONG_MAX; if( FD_LIKELY( tile->rpc.snapshot_server_enabled ) ) { @@ -1872,7 +1957,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->bundle.url_len = strnlen( tile->bundle.url, sizeof(tile->bundle.url)-1UL ); fd_cstr_ncpy( tile->bundle.sni, config->tiles.bundle.tls_domain_name, sizeof(tile->bundle.sni) ); tile->bundle.sni_len = strnlen( tile->bundle.sni, sizeof(tile->bundle.sni)-1UL ); - fd_cstr_ncpy( tile->bundle.identity_key_path, config->paths.identity_key, sizeof(tile->bundle.identity_key_path) ); + fd_cstr_ncpy( tile->bundle.identity_key_path, identity_key_path, sizeof(tile->bundle.identity_key_path) ); fd_cstr_ncpy( tile->bundle.key_log_path, config->development.bundle.ssl_key_log_file, sizeof(tile->bundle.key_log_path) ); tile->bundle.buf_sz = config->development.bundle.buffer_size_kib<<10; tile->bundle.out_depth = config->tiles.verify.receive_buffer_size; diff --git a/src/app/shared/commands/run/run.c b/src/app/shared/commands/run/run.c index 24b57b8e34f..862d001f097 100644 --- a/src/app/shared/commands/run/run.c +++ b/src/app/shared/commands/run/run.c @@ -27,6 +27,7 @@ #include "../../../../disco/sleep/fd_sleep.h" #include "../../../../util/pod/fd_pod_format.h" #include "../../../../discof/restore/utils/fd_wfs.h" +#include "../../../../disco/keyguard/fd_keyload.h" #include "../configure/configure.h" #include "../configure/fd_cpu_isolation.h" @@ -80,6 +81,8 @@ run_cmd_perm( args_t * args, fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for serving metrics" ); if( FD_UNLIKELY( config->tiles.gui.gui_listen_port<1024 ) ) fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for serving the GUI" ); + if( FD_UNLIKELY( config->firedancer.failover.enabled && config->paths.vote_account[ 0 ] && config->firedancer.failover.listen_port<1024 ) ) + fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for the failover listener" ); } struct pidns_clone_args { @@ -275,6 +278,11 @@ execve_tile( char const * name, pid_t child = fork(); if( FD_UNLIKELY( -1==child ) ) FD_LOG_ERR(( "fork() failed (%i-%s)", errno, fd_io_strerror( errno ) )); if( FD_LIKELY( !child ) ) { + if( !strcmp( tile->name, "sign" ) && tile->sign.failover_junk_fd!=-1 ) { + if( FD_UNLIKELY( -1==fcntl( tile->sign.failover_junk_fd, F_SETFD, 0 ) ) ) + FD_LOG_ERR(( "passing the failover junk key pipe failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } + char _current_executable_path[ PATH_MAX ]; FD_TEST( -1!=fd_file_util_self_exe( _current_executable_path ) ); @@ -537,6 +545,12 @@ main_pid_namespace( void * _args ) { if( FD_UNLIKELY( fd_cpuset_setaffinity( 0, floating_cpu_set ) ) ) FD_LOG_ERR(( "fd_cpuset_setaffinity failed (%i-%s)", errno, fd_io_strerror( errno ) )); + for( ulong i=0UL; itopo.tile_cnt; i++ ) { + fd_topo_tile_t const * tile = &config->topo.tiles[ i ]; + if( strcmp( tile->name, "sign" ) || tile->sign.failover_junk_fd==-1 ) continue; + if( FD_UNLIKELY( close( tile->sign.failover_junk_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } + if( FD_UNLIKELY( close( config_memfd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); if( need_xdp ) { for( uint i=0U; itopo.tile_cnt; i++ ) { + fd_topo_tile_t const * tile = &config->topo.tiles[ i ]; + if( strcmp( tile->name, "sign" ) || tile->sign.failover_junk_fd==-1 ) continue; + if( FD_UNLIKELY( close( tile->sign.failover_junk_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } if( FD_UNLIKELY( close( pipefd[ 1 ] ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); @@ -1066,14 +1085,42 @@ fdctl_check_configure( config_t const * config ) { fd_cpu_isolation_check_cgroup( config ); } +/* Each Sign tile gets its own one-shot pipe for receiving the ephemeral + generated junk key. */ +static void FD_FN_SENSITIVE +failover_junk_prepare( config_t * config ) { + if( fd_topo_find_tile( &config->topo, "failov", 0UL )==ULONG_MAX ) return; + uchar const * key = fd_keyload_generate(); + fd_memcpy( config->topo.failover_junk, key+32UL, 32UL ); + for( ulong i=0UL; itopo.tile_cnt; i++ ) { + fd_topo_tile_t * tile = &config->topo.tiles[ i ]; + if( strcmp( tile->name, "sign" ) ) continue; + FD_TEST( tile->sign.failover_enabled && tile->sign.failover_junk_fd==-1 ); + int pipefd[ 2 ]; + if( FD_UNLIKELY( pipe2( pipefd, O_CLOEXEC ) ) ) FD_LOG_ERR(( "pipe2() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + for( ulong off=0UL; off<64UL; ) { + long n = write( pipefd[ 1 ], key+off, 64UL-off ); + if( FD_UNLIKELY( n==-1L && errno==EINTR ) ) continue; + if( FD_UNLIKELY( n<=0L ) ) FD_LOG_ERR(( "could not deliver the failover junk key" )); + off += (ulong)n; + } + if( FD_UNLIKELY( close( pipefd[ 1 ] ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + tile->sign.failover_junk_fd = pipefd[ 0 ]; + } + fd_keyload_unload( key, 0 ); +} + void run_firedancer_init( config_t * config, int init_workspaces, int check_configure ) { struct stat st; int err = stat( config->paths.identity_key, &st ); - if( FD_UNLIKELY( -1==err && errno==ENOENT ) ) FD_LOG_ERR(( "[consensus.identity_path] key does not exist `%s`. You can generate an identity key at this path by running `%s keys new %s --config `", config->paths.identity_key, FD_BINARY_NAME, config->paths.identity_key )); - else if( FD_UNLIKELY( -1==err ) ) FD_LOG_ERR(( "could not stat [consensus.identity_path] `%s` (%i-%s)", config->paths.identity_key, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( -1==err && errno==ENOENT ) ) FD_LOG_ERR(( "[paths.identity_key] key does not exist `%s`. You can generate an identity key at this path by running `%s keys new %s --config `", config->paths.identity_key, FD_BINARY_NAME, config->paths.identity_key )); + else if( FD_UNLIKELY( -1==err ) ) FD_LOG_ERR(( "could not stat [paths.identity_key] `%s` (%i-%s)", config->paths.identity_key, errno, fd_io_strerror( errno ) )); + + /* Failover needs a fresh in-memory junk key. */ + failover_junk_prepare( config ); /* FIXME: fdctl_check_configure unconditionally checks for network stack prerequisites even if the command being run does not diff --git a/src/app/shared/commands/run/run1.c b/src/app/shared/commands/run/run1.c index 43b4cde9d20..e43c7afa87f 100644 --- a/src/app/shared/commands/run/run1.c +++ b/src/app/shared/commands/run/run1.c @@ -93,6 +93,9 @@ run1_cmd_fn( args_t * args, int flags = config->development.sandbox ? CLONE_NEWPID : 0; pid_t clone_pid = clone( tile_main, (uchar *)stack + FD_TILE_PRIVATE_STACK_SZ, flags, &clone_args ); if( FD_UNLIKELY( clone_pid<0 ) ) FD_LOG_ERR(( "clone() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( !strcmp( tile->name, "sign" ) && tile->sign.failover_junk_fd!=-1 ) { + if( FD_UNLIKELY( close( tile->sign.failover_junk_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } if( FD_LIKELY( args->run1.pipe_fd!=-1 ) ) { ulong pid = (ulong)clone_pid; diff --git a/src/app/shared/fd_action.h b/src/app/shared/fd_action.h index 6ece0a79db4..b301dc99f7e 100644 --- a/src/app/shared/fd_action.h +++ b/src/app/shared/fd_action.h @@ -61,6 +61,16 @@ union fdctl_args { char name[ 64UL ]; } get_identity; + struct { + char name[ 64UL ]; + char keypair_path[ PATH_MAX ]; + int cmd; + int yes; + int force; + uint addr; + ushort port; + } failover; + struct { int clean; } ps; diff --git a/src/app/shared/fd_config.c b/src/app/shared/fd_config.c index 12d3eaad4c6..f7f97239e37 100644 --- a/src/app/shared/fd_config.c +++ b/src/app/shared/fd_config.c @@ -519,9 +519,27 @@ fd_config_validatef( fd_configf_t const * config ) { CFG_HAS_NON_ZERO( runtime.program_cache_size_mib ); } +static void +fd_config_validate_failover( fd_config_t const * config ) { + fd_configf_t const * f = &config->firedancer; + + if( FD_UNLIKELY( f->failover.boot_to_standby && ( !f->failover.enabled || !strcmp( config->paths.vote_account, "" ) ) ) ) { + FD_LOG_ERR(( "[failover.boot_to_standby] is true, so [failover.enabled] must be true and [paths.vote_account] must be set" )); + } + if( FD_LIKELY( !f->failover.enabled ) ) return; + if( FD_UNLIKELY( !f->failover.listen_port ) ) { + FD_LOG_ERR(( "[failover.listen_port] must not be zero" )); + } + uint listen_addr; + if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( f->failover.listen_address, &listen_addr ) ) ) { + FD_LOG_ERR(( "[failover.listen_address] `%s` must be an IPv4 address, host names are not accepted", f->failover.listen_address )); + } +} + void fd_config_validate( fd_config_t const * config ) { fd_config_validatef( &config->firedancer ); + fd_config_validate_failover( config ); CFG_HAS_NON_EMPTY( name ); CFG_HAS_NON_EMPTY( paths.base ); diff --git a/src/app/shared/fd_config.h b/src/app/shared/fd_config.h index 86a4126188f..5058e4a1e0f 100644 --- a/src/app/shared/fd_config.h +++ b/src/app/shared/fd_config.h @@ -126,6 +126,13 @@ struct fd_configf { char authorized_voter_paths[ FD_KEYGUARD_AUTH_VOTERS_MAX ][ PATH_MAX ]; } paths; + struct { + int enabled; + int boot_to_standby; + char listen_address[ 16 ]; + ushort listen_port; + } failover; + }; typedef struct fd_configf fd_configf_t; diff --git a/src/app/shared/fd_config_json.c b/src/app/shared/fd_config_json.c index 56129762d1a..0289f2fb3af 100644 --- a/src/app/shared/fd_config_json.c +++ b/src/app/shared/fd_config_json.c @@ -12,7 +12,7 @@ or knowingly skipped) before the constant is bumped. String keys of the user's own file are separately forced through the classification lists below. */ -FD_STATIC_ASSERT( sizeof(fd_config_t)==24458368UL, update_fd_config_to_json_for_the_layout_change ); +FD_STATIC_ASSERT( sizeof(fd_config_t)==24462528UL, update_fd_config_to_json_for_the_layout_change ); #define REDACTED "[redacted]" @@ -126,6 +126,7 @@ static char const * const jw_redacted_keys[] = { "snapshots.sources.servers", "snapshots.server.http_listen_address", "hugetlbfs.mount_path", + "failover.listen_address", "net.bind_address", "tiles.quic.ssl_key_log_file", "tiles.bundle.url", @@ -419,6 +420,13 @@ fd_config_to_json( fd_config_t const * config, jw_obj_close( &w ); jw_obj_close( &w ); + jw_obj_open( &w, "failover" ); + jw_bool ( &w, "enabled", f->failover.enabled ); + jw_bool ( &w, "boot_to_standby", f->failover.boot_to_standby ); + jw_path ( &w, "listen_address", f->failover.listen_address ); + jw_ulong( &w, "listen_port", f->failover.listen_port ); + jw_obj_close( &w ); + jw_obj_open( &w, "hugetlbfs" ); jw_path ( &w, "gigantic_page_mount_path", config->hugetlbfs.gigantic_page_mount_path ); jw_path ( &w, "huge_page_mount_path", config->hugetlbfs.huge_page_mount_path ); diff --git a/src/app/shared/fd_config_parse.c b/src/app/shared/fd_config_parse.c index ad342d3494a..ecb3542b877 100644 --- a/src/app/shared/fd_config_parse.c +++ b/src/app/shared/fd_config_parse.c @@ -77,6 +77,11 @@ fd_config_extract_podf( uchar * pod, CFG_POP ( ulong, snapshots.server.send_timeout_millis ); CFG_POP ( ulong, snapshots.server.send_buffer_size_kib ); + CFG_POP ( bool, failover.enabled ); + CFG_POP ( bool, failover.boot_to_standby ); + CFG_POP ( cstr, failover.listen_address ); + CFG_POP ( ushort, failover.listen_port ); + CFG_POP ( bool, development.hard_fork_fatal ); CFG_POP ( bool, development.fixed_fec_sets ); diff --git a/src/app/shared/test_config_json.c b/src/app/shared/test_config_json.c index a38d143381a..fa963ac72a1 100644 --- a/src/app/shared/test_config_json.c +++ b/src/app/shared/test_config_json.c @@ -139,6 +139,16 @@ main( int argc, FD_TEST( !strstr( json, "hunter2" ) ); FD_TEST( !strstr( json, "example.com" ) ); + /* failover settings are reported, failover is on by default */ + FD_TEST( strstr( json, "\"failover\":{\"enabled\":true,\"boot_to_standby\":false,\"listen_address\":\"[redacted]\",\"listen_port\":8010}" ) ); + + /* the listen address is redacted, the rest is reported */ + config->firedancer.failover.enabled = 0; + strcpy( config->firedancer.failover.listen_address, "10.9.8.7" ); + FD_TEST( fd_config_to_json( config, json, sizeof(json) ) ); + FD_TEST( !strstr( json, "10.9.8.7" ) ); + FD_TEST( strstr( json, "\"failover\":{\"enabled\":false,\"boot_to_standby\":false,\"listen_address\":\"[redacted]\",\"listen_port\":8010}" ) ); + /* user override toml renders with keyword redaction */ static char const user_toml[] = "name = \"fd1\"\n" diff --git a/src/app/shared/test_config_parse.c b/src/app/shared/test_config_parse.c index ccbf91fa08a..9035e91a35c 100644 --- a/src/app/shared/test_config_parse.c +++ b/src/app/shared/test_config_parse.c @@ -31,6 +31,12 @@ static char const cfg_str_5[] = static char const cfg_str_vote_history[] = "[paths]\n" " vote_history = \"/data/{name}/vote_history\"\n"; +static char const cfg_str_failover[] = + "[failover]\n" + " enabled = true\n" + " boot_to_standby = true\n" + " listen_address = \"10.0.0.1\"\n" + " listen_port = 9010"; extern uchar const firedancer_default_config[]; extern ulong const firedancer_default_config_sz; @@ -99,6 +105,34 @@ consensus_is_valid( ulong wfs_slot, return WIFEXITED( status ) && !WEXITSTATUS( status ); } +/* Validates the config with the given failover knobs in a child, so the + FD_LOG_ERR paths can be checked. */ + +static int +failover_is_valid( config_t * config, + int enabled, + int boot_to_standby, + char const * listen_address, + ushort listen_port, + char const * vote_account ) { + int pid = fork(); + FD_TEST( pid>=0 ); + if( FD_UNLIKELY( !pid ) ) { + config->firedancer.development.genesis.max_file_size_mib = 4055UL; + config->firedancer.failover.enabled = enabled; + config->firedancer.failover.boot_to_standby = boot_to_standby; + config->firedancer.failover.listen_port = listen_port; + fd_cstr_ncpy( config->firedancer.failover.listen_address, listen_address, sizeof(config->firedancer.failover.listen_address) ); + fd_cstr_ncpy( config->paths.vote_account, vote_account, sizeof(config->paths.vote_account) ); + fd_config_validate( config ); + _exit( 0 ); + } + + int status = 0; + FD_TEST( waitpid( pid, &status, 0 )==pid ); + return WIFEXITED( status ) && !WEXITSTATUS( status ); +} + int main( int argc, char ** argv ) { @@ -204,6 +238,18 @@ main( int argc, FD_TEST( genesis_max_file_size_is_valid( config, 4055UL ) ); FD_TEST( !genesis_max_file_size_is_valid( config, 4056UL ) ); + FD_TEST( failover_is_valid( config, 1, 0, "0.0.0.0", 8010, "vote.json" ) ); + FD_TEST( failover_is_valid( config, 1, 0, "10.0.0.1", 8010, "vote.json" ) ); + FD_TEST( !failover_is_valid( config, 1, 0, "0.0.0.0", 0, "vote.json" ) ); + FD_TEST( !failover_is_valid( config, 1, 0, "localhost", 8010, "vote.json" ) ); + FD_TEST( failover_is_valid( config, 1, 0, "0.0.0.0", 8010, "" ) ); /* inactive without a vote account */ + FD_TEST( failover_is_valid( config, 0, 0, "", 0, "" ) ); + /* boot_to_standby needs failover enabled and a vote account */ + FD_TEST( failover_is_valid( config, 1, 1, "0.0.0.0", 8010, "vote.json" ) ); + FD_TEST( !failover_is_valid( config, 1, 1, "0.0.0.0", 8010, "" ) ); + FD_TEST( !failover_is_valid( config, 0, 1, "0.0.0.0", 8010, "vote.json" ) ); + FD_TEST( !failover_is_valid( config, 0, 1, "", 0, "" ) ); + /* Ensure we can selectively override a field */ config->gossip.port = 9191; @@ -280,6 +326,17 @@ main( int argc, # undef HASH + /* Parse the failover knobs */ + + fd_memset( config, 0, sizeof(config_t) ); + pod = fd_pod_join( fd_pod_new( pod_mem, sizeof(pod_mem) ) ); + FD_TEST( fd_toml_parse( cfg_str_failover, sizeof(cfg_str_failover)-1, pod, scratch, sizeof(scratch), NULL ) == FD_TOML_SUCCESS ); + FD_TEST( fd_config_extract_pod( pod, config ) == config ); + FD_TEST( config->firedancer.failover.enabled ); + FD_TEST( config->firedancer.failover.boot_to_standby ); + FD_TEST( !strcmp( config->firedancer.failover.listen_address, "10.0.0.1" ) ); + FD_TEST( config->firedancer.failover.listen_port==9010 ); + FD_LOG_NOTICE(( "pass" )); fd_halt(); } diff --git a/src/choreo/votor/Local.mk b/src/choreo/votor/Local.mk index 8e5cfe59d98..8af8d311789 100644 --- a/src/choreo/votor/Local.mk +++ b/src/choreo/votor/Local.mk @@ -1,5 +1,5 @@ -$(call add-hdrs,ag_votor_base.h ag_bls_serde.h ag_vote.h ag_vote_serde.h ag_epoch_info.h ag_cert.h ag_cert_serde.h ag_vote_history_serde.h ag_pool.h ag_slot_state.h ag_finality_tracker.h ag_parent_ready_tracker.h ag_votor.h) -$(call add-objs,ag_bls_serde ag_vote ag_vote_serde ag_epoch_info ag_cert ag_cert_serde ag_pool ag_slot_state ag_finality_tracker ag_parent_ready_tracker ag_votor ag_vote_history_serde,fd_choreo) +$(call add-hdrs,ag_votor_base.h ag_bls_serde.h ag_vote.h ag_vote_serde.h ag_epoch_info.h ag_cert.h ag_cert_serde.h ag_vote_history_serde.h ag_hist.h ag_pool.h ag_slot_state.h ag_finality_tracker.h ag_parent_ready_tracker.h ag_votor.h) +$(call add-objs,ag_bls_serde ag_vote ag_vote_serde ag_epoch_info ag_cert ag_cert_serde ag_pool ag_slot_state ag_finality_tracker ag_parent_ready_tracker ag_votor ag_vote_history_serde ag_hist,fd_choreo) ifdef FD_HAS_HOSTED $(call make-unit-test,test_ag_votor_base,test_ag_votor_base,fd_util) @@ -7,6 +7,7 @@ $(call make-unit-test,test_ag_bls,test_ag_bls,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_vote,test_ag_vote,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_epoch_info,test_ag_epoch_info,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_cert,test_ag_cert,fd_choreo fd_ballet fd_util) +$(call make-unit-test,test_ag_hist,test_ag_hist,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_pool,test_ag_pool,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_slot_state,test_ag_slot_state,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_finality_tracker,test_ag_finality_tracker,fd_choreo fd_ballet fd_util) @@ -14,15 +15,18 @@ $(call make-unit-test,test_ag_parent_ready_tracker,test_ag_parent_ready_tracker, $(call make-unit-test,test_ag_votor,test_ag_votor,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_vote_history_serde,test_ag_vote_history_serde,fd_choreo fd_ballet fd_util) $(call run-unit-test,test_ag_vote_history_serde) +$(call make-unit-test,test_ag_votor_hist,test_ag_votor_hist,fd_choreo fd_ballet fd_util) $(call run-unit-test,test_ag_votor_base) $(call run-unit-test,test_ag_bls) $(call run-unit-test,test_ag_vote) $(call run-unit-test,test_ag_epoch_info) $(call run-unit-test,test_ag_cert) +$(call run-unit-test,test_ag_hist) $(call run-unit-test,test_ag_pool) $(call run-unit-test,test_ag_slot_state) $(call run-unit-test,test_ag_finality_tracker) $(call run-unit-test,test_ag_parent_ready_tracker) $(call run-unit-test,test_ag_votor) +$(call run-unit-test,test_ag_votor_hist) $(call make-fuzz-test,fuzz_ag_vote_history_serde,fuzz_ag_vote_history_serde,fd_choreo fd_ballet fd_util) endif diff --git a/src/choreo/votor/ag_hist.c b/src/choreo/votor/ag_hist.c new file mode 100644 index 00000000000..fc160186713 --- /dev/null +++ b/src/choreo/votor/ag_hist.c @@ -0,0 +1,87 @@ +#include "ag_hist.h" + +/* rec_sz is the wire size of a record with these flags */ + +FD_FN_CONST static inline ulong +rec_sz( uchar flags ) { + return AG_HIST_REC_MIN_SZ + fd_ulong_if( flags & AG_HIST_FLAG_VOTED_NOTAR, sizeof(ag_block_hash_t), 0UL ); +} + +/* hist_ok checks the rules both ag_hist_ser and ag_hist_de enforce. */ + +static int +hist_ok( ag_hist_t const * hist ) { + if( FD_UNLIKELY( hist->rec_cnt>AG_HIST_MAX ) ) return 0; + if( FD_UNLIKELY( hist->anchor==ULONG_MAX ) ) return 0; + ulong first = ag_hist_first_slot( hist->anchor ); + ulong prev = ULONG_MAX; + for( ulong i=0UL; irec_cnt; i++ ) { + ag_hist_rec_t const * rec = &hist->rec[ i ]; + if( FD_UNLIKELY( rec->slotslot<=prev ) ) return 0; + if( FD_UNLIKELY( rec->flags & ~AG_HIST_FLAG_MASK ) ) return 0; + if( FD_UNLIKELY( !( rec->flags & AG_HIST_FLAG_VOTED ) ) ) return 0; + prev = rec->slot; + } + return 1; +} + +int +ag_hist_ser( ag_hist_t const * hist, + uchar * buf, + ulong buf_max, + ulong * out_sz ) { + if( FD_UNLIKELY( !hist_ok( hist ) ) ) return -1; + + ulong sz = AG_HIST_HDR_SZ; + for( ulong i=0UL; irec_cnt; i++ ) sz += rec_sz( hist->rec[ i ].flags ); + if( FD_UNLIKELY( sz>buf_max ) ) return -1; + + ulong off = 0UL; + FD_STORE( ulong, buf+off, hist->anchor ); off += sizeof(ulong); + FD_STORE( ulong, buf+off, hist->last_leader_slot ); off += sizeof(ulong); + FD_STORE( ulong, buf+off, hist->vote_bound ); off += sizeof(ulong); + FD_STORE( ushort, buf+off, (ushort)hist->rec_cnt ); off += sizeof(ushort); + for( ulong i=0UL; irec_cnt; i++ ) { + ag_hist_rec_t const * rec = &hist->rec[ i ]; + FD_STORE( ulong, buf+off, rec->slot ); off += sizeof(ulong); + buf[ off ] = rec->flags; off += sizeof(uchar); + if( rec->flags & AG_HIST_FLAG_VOTED_NOTAR ) { fd_memcpy( buf+off, rec->notar_hash, sizeof(ag_block_hash_t) ); off += sizeof(ag_block_hash_t); } + } + + *out_sz = off; + return 0; +} + +int +ag_hist_de( uchar const * buf, + ulong buf_sz, + ag_hist_t * out ) { + if( FD_UNLIKELY( buf_szanchor = FD_LOAD( ulong, buf+off ); off += sizeof(ulong); + hist->last_leader_slot = FD_LOAD( ulong, buf+off ); off += sizeof(ulong); + hist->vote_bound = FD_LOAD( ulong, buf+off ); off += sizeof(ulong); + hist->rec_cnt = FD_LOAD( ushort, buf+off ); off += sizeof(ushort); + if( FD_UNLIKELY( hist->rec_cnt>AG_HIST_MAX ) ) return -1; + + for( ulong i=0UL; irec_cnt; i++ ) { + ag_hist_rec_t * rec = &hist->rec[ i ]; + if( FD_UNLIKELY( buf_sz-offslot = FD_LOAD( ulong, buf+off ); off += sizeof(ulong); + rec->flags = buf[ off ]; off += sizeof(uchar); + if( rec->flags & AG_HIST_FLAG_VOTED_NOTAR ) { + if( FD_UNLIKELY( buf_sz-offnotar_hash, buf+off, sizeof(ag_block_hash_t) ); off += sizeof(ag_block_hash_t); + } + } + if( FD_UNLIKELY( off!=buf_sz ) ) return -1; /* trailing bytes */ + if( FD_UNLIKELY( !hist_ok( hist ) ) ) return -1; + + *out = *hist; + return 0; +} diff --git a/src/choreo/votor/ag_hist.h b/src/choreo/votor/ag_hist.h new file mode 100644 index 00000000000..54fc4fbd90c --- /dev/null +++ b/src/choreo/votor/ag_hist.h @@ -0,0 +1,76 @@ +#ifndef HEADER_fd_src_choreo_votor_ag_hist_h +#define HEADER_fd_src_choreo_votor_ag_hist_h + +/* ag_hist serializes and deserializes a validator's own Alpenglow votes + since a finality anchor, so another validator can adopt the same + consensus state before voting as the same identity. */ + +#include "ag_votor_base.h" + +#define AG_HIST_MAX (128UL) /* records per history, 32 leader windows */ + +#define AG_HIST_FLAG_VOTED (1U) /* any own vote on the slot */ +#define AG_HIST_FLAG_VOTED_NOTAR (2U) /* a notar vote, notar_hash is set */ +#define AG_HIST_FLAG_BAD_WINDOW (4U) /* no final vote may follow on this slot */ +#define AG_HIST_FLAG_RETIRED (8U) /* the slot is done */ +#define AG_HIST_FLAG_MASK (15U) + +#define AG_HIST_HDR_SZ (26UL) /* anchor u64, last_leader_slot u64, vote_bound u64, rec_cnt u16 */ +#define AG_HIST_REC_MIN_SZ (9UL) /* slot u64, flags u8 */ +#define AG_HIST_REC_MAX_SZ (41UL) /* plus the 32 byte notar hash */ +#define AG_HIST_SER_MAX (AG_HIST_HDR_SZ+AG_HIST_MAX*AG_HIST_REC_MAX_SZ) /* 5274 */ + +struct ag_hist_rec { + ulong slot; + uchar flags; /* AG_HIST_FLAG_* */ + ag_block_hash_t notar_hash; /* only meaningful when AG_HIST_FLAG_VOTED_NOTAR is set */ +}; +typedef struct ag_hist_rec ag_hist_rec_t; + +struct ag_hist { + ulong anchor; /* exporter's highest_final_cert_slot, raised when the export was truncated */ + ulong last_leader_slot; /* slot of the last LEADER the exporter published, ULONG_MAX when none */ + ulong vote_bound; /* the exporter casts no vote at or below this slot, ULONG_MAX when none */ + ulong rec_cnt; + ag_hist_rec_t rec[ AG_HIST_MAX ]; /* ascending slots */ +}; +typedef struct ag_hist ag_hist_t; + +FD_PROTOTYPES_BEGIN + +/* ag_hist_first_slot returns the lowest slot a history with this anchor + may contain. */ + +FD_FN_CONST static inline ulong +ag_hist_first_slot( ulong anchor ) { + return ag_first_slot_in_window( fd_ulong_sat_sub( anchor, AG_REWARD_SLOT_DELTA ) ); +} + +/* ag_hist_ser serializes hist into buf. Returns 0 on success, -1 if hist + is invalid or buf_max is too small. */ + +int +ag_hist_ser( ag_hist_t const * hist, + uchar * buf, + ulong buf_max, + ulong * out_sz ); + +/* ag_hist_de deserializes exactly buf_sz bytes of buf into out. Returns 0 + on success, -1 if the bytes are not a valid history. */ + +int +ag_hist_de( uchar const * buf, + ulong buf_sz, + ag_hist_t * out ); + +/* ag_hist_tip returns the highest record slot, ULONG_MAX when the + history is empty. */ + +FD_FN_PURE static inline ulong +ag_hist_tip( ag_hist_t const * hist ) { + return hist->rec_cnt ? hist->rec[ hist->rec_cnt-1UL ].slot : ULONG_MAX; +} + +FD_PROTOTYPES_END + +#endif /* HEADER_fd_src_choreo_votor_ag_hist_h */ diff --git a/src/choreo/votor/ag_votor.c b/src/choreo/votor/ag_votor.c index 1045bbaad09..67eb8c0e528 100644 --- a/src/choreo/votor/ag_votor.c +++ b/src/choreo/votor/ag_votor.c @@ -90,6 +90,7 @@ struct __attribute__((aligned(128UL))) ag_votor { fd_bls_sign_fn bls_sign_fn; void * bls_sign_ctx; + ulong vote_bound; /* we cast no vote at or below this slot, ULONG_MAX when none */ epoch_bls_key_t prev_epoch; epoch_bls_key_t curr_epoch; @@ -276,6 +277,7 @@ ag_votor_new( void * mem, votor->prev_epoch = (epoch_bls_key_t){ .start_slot = ULONG_MAX, .rank = USHORT_MAX }; votor->curr_epoch = (epoch_bls_key_t){ .start_slot = ULONG_MAX, .rank = USHORT_MAX }; votor->next_epoch = (epoch_bls_key_t){ .start_slot = ULONG_MAX, .rank = USHORT_MAX }; + votor->vote_bound = ULONG_MAX; votor->vote_events = vote_events_join( vote_events_new( vote_events, events_max ) ); votor->pending_dlist = pending_dlist_join( pending_dlist_new( pending_dlist ) ); for( ulong k=0UL; ktimeout_dlist[ k ] = timeout_dlist_join( timeout_dlist_new( timeout_dlist+k*timeout_dlist_footprint() ) ); @@ -403,6 +405,14 @@ first_unpruned_slot( ag_votor_t const * self ) { return ag_first_slot_in_window( fd_ulong_sat_sub( self->highest_final_cert_slot, AG_REWARD_SLOT_DELTA ) ); } +/* Returns whether slot is at or below our vote bound. */ + +FD_FN_PURE static int +below_vote_bound( ag_votor_t const * self, + ulong slot ) { + return self->vote_bound!=ULONG_MAX && slot<=self->vote_bound; +} + static int should_ignore_pool_event( ag_votor_t const * self, ag_pool_event_t const * event ) { @@ -428,6 +438,7 @@ try_final( ag_votor_t * self, ulong slot, ag_block_hash_t const hash ) { FD_TEST( slot>=first_unpruned_slot( self ) ); + if( FD_UNLIKELY( below_vote_bound( self, slot ) ) ) return; epoch_bls_key_t const * epoch = own_epoch( self, slot ); slot_state_ele_t const * state = slot_state_map_ele_query_const( self->slot_states->map, &slot, NULL, self->slot_states->pool ); @@ -450,6 +461,7 @@ try_notar( ag_votor_t * self, FD_TEST( slot>=first_unpruned_slot( self ) ); epoch_bls_key_t const * epoch = own_epoch( self, slot ); if( FD_UNLIKELY( has_voted( self, slot ) ) ) return 0; + if( FD_UNLIKELY( below_vote_bound( self, slot ) ) ) return 0; ag_block_hash_t hash; memcpy( hash, block_info->hash, sizeof(ag_block_hash_t) ); @@ -489,6 +501,7 @@ try_skip_window( ag_votor_t * self, ulong window_start = ag_first_slot_in_window( slot ); for( ulong s=window_start; svoted = 1; @@ -605,6 +618,127 @@ epoch_starting_at( ag_votor_t * self, FD_LOG_CRIT(( "no epoch starts at slot %lu", epoch_slot )); } +void +ag_votor_set_vote_bound( ag_votor_t * self, + ulong bound ) { + self->vote_bound = self->vote_bound==ULONG_MAX ? bound : fd_ulong_max( self->vote_bound, bound ); +} + +ulong +ag_votor_get_vote_bound( ag_votor_t const * self ) { + return self->vote_bound; +} + +/* advance_root raises the finality anchor to slot and prunes below it, if + slot is past the anchor. */ + +static void +advance_root( ag_votor_t * self, + ulong slot ) { + if( FD_UNLIKELY( self->highest_final_cert_slot==ULONG_MAX || slot<=self->highest_final_cert_slot ) ) return; + self->highest_final_cert_slot = slot; + prune( self ); +} + +int +ag_votor_hist_export( ag_votor_t * self, + ulong last_leader_slot, + ag_hist_t * out ) { + out->last_leader_slot = last_leader_slot; + out->vote_bound = self->vote_bound; + out->rec_cnt = 0UL; + if( FD_UNLIKELY( self->highest_final_cert_slot==ULONG_MAX ) ) { + out->anchor = 0UL; + return 0; + } + + slot_state_map_t * map = self->slot_states->map; + slot_state_ele_t * pool = self->slot_states->pool; + ulong * slots = self->scratch.slots; + ulong first = first_unpruned_slot( self ); + ulong cnt = 0UL; + for( slot_state_map_iter_t iter = slot_state_map_iter_init( map, pool ); + !slot_state_map_iter_done( iter, map, pool ); + iter = slot_state_map_iter_next( iter, map, pool ) ) { + slot_state_ele_t const * ele = slot_state_map_iter_ele_const( iter, map, pool ); + if( FD_LIKELY( ele->voted && ele->slot>=first && cntslot_max ) ) slots[ cnt++ ] = ele->slot; + } + slot_sort_inplace( slots, cnt ); + + /* If there are too many voted slots, we drop whole windows from the + bottom and raise the anchor past them. */ + ulong lo = 0UL; + while( cnt-lo>AG_HIST_MAX ) { + ulong next_window = ag_first_slot_in_window( slots[ lo ] )+AG_SLOTS_PER_WINDOW; + while( lo0UL; + out->anchor = self->highest_final_cert_slot; + if( FD_UNLIKELY( truncated ) ) out->anchor = fd_ulong_max( out->anchor, ag_first_slot_in_window( slots[ lo ] )+AG_REWARD_SLOT_DELTA ); + + for( ulong i=lo; irec[ out->rec_cnt++ ]; + rec->slot = ele->slot; + rec->flags = (uchar)( AG_HIST_FLAG_VOTED | + fd_uint_if( ele->voted_notar, AG_HIST_FLAG_VOTED_NOTAR, 0U ) | + fd_uint_if( ele->bad_window, AG_HIST_FLAG_BAD_WINDOW, 0U ) | + fd_uint_if( ele->retired, AG_HIST_FLAG_RETIRED, 0U ) ); + if( ele->voted_notar ) fd_memcpy( rec->notar_hash, ele->voted_notar_hash, sizeof(ag_block_hash_t) ); + else fd_memset( rec->notar_hash, 0, sizeof(ag_block_hash_t) ); + } + return truncated; +} + +ulong +ag_votor_hist_adopt( ag_votor_t * self, + ag_hist_t const * hist ) { + slot_state_map_t * map = self->slot_states->map; + slot_state_ele_t * pool = self->slot_states->pool; + ulong first = ag_hist_first_slot( fd_ulong_max( self->highest_final_cert_slot, hist->anchor ) ); + ulong free = slot_state_pool_free( pool ); + if( hist->anchor>self->highest_final_cert_slot ) { + for( slot_state_map_iter_t iter = slot_state_map_iter_init( map, pool ); + !slot_state_map_iter_done( iter, map, pool ); + iter = slot_state_map_iter_next( iter, map, pool ) ) { + ulong slot = slot_state_map_iter_ele_const( iter, map, pool )->slot; + free += slot>=self->root && slotrec_cnt; i++ ) { + ulong slot = hist->rec[ i ].slot; + if( slotanchor ); + /* Later pruning must visit records imported below the startup cursor. */ + self->root = fd_ulong_min( self->root, first ); + /* Take the exporter's vote bound as well. */ + if( FD_UNLIKELY( hist->vote_bound!=ULONG_MAX ) ) ag_votor_set_vote_bound( self, hist->vote_bound ); + ulong conflicts = 0UL; + for( ulong i=0UL; irec_cnt; i++ ) { + ag_hist_rec_t const * rec = &hist->rec[ i ]; + if( FD_UNLIKELY( rec->slotslot ); + if( rec->flags & AG_HIST_FLAG_VOTED_NOTAR ) { + if( FD_UNLIKELY( state->voted_notar && !fd_memeq( state->voted_notar_hash, rec->notar_hash, sizeof(ag_block_hash_t) ) ) ) conflicts++; + state->voted_notar = 1; + fd_memcpy( state->voted_notar_hash, rec->notar_hash, sizeof(ag_block_hash_t) ); + } + if( !state->voted ) { + if( FD_UNLIKELY( state->pending_block ) ) pending_dlist_ele_remove( self->pending_dlist, state, self->slot_states->pool ); + state->pending_block = 0; + state->voted = 1; + } + state->bad_window |= !!( rec->flags & AG_HIST_FLAG_BAD_WINDOW ); + state->retired |= !!( rec->flags & AG_HIST_FLAG_RETIRED ); + } + return conflicts; +} + void ag_votor_set_bls_key( ag_votor_t * self, ulong epoch_slot, @@ -664,6 +798,8 @@ ag_votor_handle_pool_event( ag_votor_t * self, case AG_POOL_EVENT_SAFE_TO_NOTAR: { ulong slot = event->safe_to_notar.slot; uchar const * hash = event->safe_to_notar.hash; + /* No fallback vote at or below the vote bound. */ + if( FD_UNLIKELY( below_vote_bound( self, slot ) ) ) { try_skip_window( self, slot, AG_VOTOR_REASON_SAFE_TO_NOTAR ); break; } epoch_bls_key_t const * epoch = own_epoch( self, slot ); if( FD_LIKELY( epoch->has_bls_key && slot>=self->catchup_vote_slot ) ) { ag_vote_t vote = ag_vote_construct_notar_fallback( self->bls_sign_fn, self->bls_sign_ctx, epoch->bls_key, slot, hash, (ushort)epoch->rank, self->shred_version ); @@ -676,6 +812,8 @@ ag_votor_handle_pool_event( ag_votor_t * self, case AG_POOL_EVENT_SAFE_TO_SKIP: { ulong slot = event->safe_to_skip; + /* No fallback vote at or below the vote bound. */ + if( FD_UNLIKELY( below_vote_bound( self, slot ) ) ) { try_skip_window( self, slot, AG_VOTOR_REASON_SAFE_TO_SKIP ); break; } epoch_bls_key_t const * epoch = own_epoch( self, slot ); if( FD_LIKELY( epoch->has_bls_key && slot>=self->catchup_vote_slot ) ) { ag_vote_t vote = ag_vote_construct_skip_fallback( self->bls_sign_fn, self->bls_sign_ctx, epoch->bls_key, slot, (ushort)epoch->rank, self->shred_version ); diff --git a/src/choreo/votor/ag_votor.h b/src/choreo/votor/ag_votor.h index ebe13ca83ce..d00302c2be5 100644 --- a/src/choreo/votor/ag_votor.h +++ b/src/choreo/votor/ag_votor.h @@ -4,6 +4,7 @@ #include "ag_votor_base.h" #include "../../ballet/bls/fd_bls.h" #include "ag_pool.h" +#include "ag_hist.h" #define AG_VOTOR_REASON_BLOCK_REPLAYED (0) #define AG_VOTOR_REASON_PARENT_READY (1) @@ -89,6 +90,34 @@ ag_votor_set_rank( ag_votor_t * self, ulong epoch_slot, ulong epoch_rank ); +/* ag_votor_set_vote_bound stops us voting at or below bound, which only + moves up. ag_votor_get_vote_bound returns it, ULONG_MAX when none is + set. */ + +void +ag_votor_set_vote_bound( ag_votor_t * self, + ulong bound ); + +FD_FN_PURE ulong +ag_votor_get_vote_bound( ag_votor_t const * self ); + +/* ag_votor_hist_export writes our own votes since the anchor into out. + Returns 1 if it had to drop old windows to fit. */ + +int +ag_votor_hist_export( ag_votor_t * self, + ulong last_leader_slot, + ag_hist_t * out ); + +/* ag_votor_hist_adopt merges a valid history into initialized slot states. + Returns how many notar hashes disagreed with ours, or ULONG_MAX if the + retained local and imported slots exceed capacity. Failure leaves + consensus state unchanged. */ + +ulong +ag_votor_hist_adopt( ag_votor_t * self, + ag_hist_t const * hist ); + /* Algorithm 1, lines 9-25. Votor::handle_pool_event */ ag_cert_t const * diff --git a/src/choreo/votor/test_ag_hist.c b/src/choreo/votor/test_ag_hist.c new file mode 100644 index 00000000000..4428dcfc1d7 --- /dev/null +++ b/src/choreo/votor/test_ag_hist.c @@ -0,0 +1,25 @@ +#include "ag_hist.h" + +int +main( int argc, char ** argv ) { + fd_boot( &argc, &argv ); + ag_hist_t hist = { .anchor=100UL, .last_leader_slot=ULONG_MAX, .vote_bound=99UL, .rec_cnt=1UL }; + hist.rec[0].slot = 101UL; + hist.rec[0].flags = AG_HIST_FLAG_VOTED|AG_HIST_FLAG_VOTED_NOTAR; + fd_memset( hist.rec[0].notar_hash, 0x42, sizeof(ag_block_hash_t) ); + uchar buf[AG_HIST_SER_MAX]; + ulong sz; + FD_TEST( !ag_hist_ser( &hist, buf, sizeof(buf), &sz ) ); + FD_TEST( sz==AG_HIST_HDR_SZ+AG_HIST_REC_MAX_SZ ); + ag_hist_t out; + FD_TEST( !ag_hist_de( buf, sz, &out ) ); + FD_TEST( out.anchor==hist.anchor && out.rec_cnt==1UL && out.last_leader_slot==ULONG_MAX && out.vote_bound==99UL ); + FD_TEST( out.rec[0].slot==101UL && out.rec[0].flags==hist.rec[0].flags ); + FD_TEST( fd_memeq( out.rec[0].notar_hash, hist.rec[0].notar_hash, sizeof(ag_block_hash_t) ) ); + ag_hist_t saved = out; + FD_TEST( ag_hist_de( buf, sz-1UL, &out )==-1 ); + FD_TEST( fd_memeq( &out, &saved, sizeof(out) ) ); + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} diff --git a/src/choreo/votor/test_ag_votor_hist.c b/src/choreo/votor/test_ag_votor_hist.c new file mode 100644 index 00000000000..e79957662b1 --- /dev/null +++ b/src/choreo/votor/test_ag_votor_hist.c @@ -0,0 +1,307 @@ +#include "ag_votor.c" +#include "test_ag_cert_builder.h" + +#define NV (2UL) +#define TEST_SLOT_MAX (16UL) +#define TEST_SHRED_VERSION ((ushort)0x5a5a) + +#define TEST_NS_PER_SLOT (400000000L) + +#define FD_TEST_NO_MSG( votor ) do { \ + ag_vote_t unused_; \ + FD_TEST( !try_recv( (votor), &unused_ ) ); \ + } while( 0 ) + +#define SCRATCH_MAX (1UL<<23) + +static uchar scratch_a[ SCRATCH_MAX ] __attribute__((aligned(128))); +static uchar scratch_b[ SCRATCH_MAX ] __attribute__((aligned(128))); + +static fd_bls_sec_t g_sk[ NV ]; +static ulong g_hash_ctr = 0UL; + +static void +genesis_hash( ag_block_hash_t out ) { + fd_memset( out, 0, sizeof(ag_block_hash_t) ); +} + +static void +random_hash( ag_block_hash_t out ) { + fd_memset( out, 0, sizeof(ag_block_hash_t) ); + FD_STORE( ulong, out, 0x9000UL + (++g_hash_ctr) ); + FD_STORE( ulong, out+8UL, 0xc0ffee00UL ^ g_hash_ctr ); +} + +static ag_block_id_t +genesis_block_id( void ) { + ag_block_id_t b; b.slot = 0UL; genesis_hash( b.hash ); + return b; +} + +static void +create_validators( void ) { + for( ulong i=0UL; irec_cnt ); + ag_hist_rec_t const * rec = &hist->rec[ idx ]; + FD_TEST( rec->slot ==slot ); + FD_TEST( rec->flags==(uchar)flags ); + if( notar_hash ) FD_TEST( fd_memeq( rec->notar_hash, notar_hash, sizeof(ag_block_hash_t) ) ); +} + +static void +round_trip( ag_hist_t const * hist ) { + uchar buf[ AG_HIST_SER_MAX ]; + ulong sz = 0UL; + FD_TEST( !ag_hist_ser( hist, buf, sizeof(buf), &sz ) ); + + ag_hist_t out; fd_memset( &out, 0xAA, sizeof(ag_hist_t) ); + FD_TEST( !ag_hist_de( buf, sz, &out ) ); + FD_TEST( out.anchor ==hist->anchor ); + FD_TEST( out.last_leader_slot==hist->last_leader_slot ); + FD_TEST( out.vote_bound ==hist->vote_bound ); + FD_TEST( out.rec_cnt ==hist->rec_cnt ); + for( ulong i=0UL; irec_cnt; i++ ) { + FD_TEST( out.rec[ i ].slot ==hist->rec[ i ].slot ); + FD_TEST( out.rec[ i ].flags==hist->rec[ i ].flags ); + if( hist->rec[ i ].flags & AG_HIST_FLAG_VOTED_NOTAR ) FD_TEST( fd_memeq( out.rec[ i ].notar_hash, hist->rec[ i ].notar_hash, sizeof(ag_block_hash_t) ) ); + } +} + +static void +test_export_shape( void ) { + ag_votor_t * votor = setup_votor( scratch_a, 0L ); + ag_block_id_t parent = genesis_block_id(); + + ag_block_hash_t hash[ 4 ]; + genesis_hash( hash[ 0 ] ); + for( ulong s=1UL; s<=3UL; s++ ) { + ag_vote_t vote = send_block_and_expect_notar( votor, s, &parent ); + fd_memcpy( hash[ s ], vote.notar.block_hash, sizeof(ag_block_hash_t) ); + parent = ag_block_id( s, hash[ s ] ); + } + FD_TEST_NO_MSG( votor ); + + ag_hist_t hist; + FD_TEST( !ag_votor_hist_export( votor, 42UL, &hist ) ); + FD_TEST( hist.anchor ==0UL ); + FD_TEST( hist.last_leader_slot==42UL ); + FD_TEST( hist.vote_bound ==ULONG_MAX ); + FD_TEST( hist.rec_cnt ==4UL ); + expect_rec( &hist, 0UL, 0UL, AG_HIST_FLAG_VOTED|AG_HIST_FLAG_VOTED_NOTAR|AG_HIST_FLAG_RETIRED, hash[ 0 ] ); + for( ulong s=1UL; s<=3UL; s++ ) expect_rec( &hist, s, s, AG_HIST_FLAG_VOTED|AG_HIST_FLAG_VOTED_NOTAR, hash[ s ] ); + round_trip( &hist ); + + teardown_votor( votor ); + FD_LOG_NOTICE(( "pass: export_shape" )); +} + +/* Test that the exporter's vote bound goes out with its history and holds + on the adopter. */ + +static void +test_bound_travels( void ) { + ag_votor_t * a = setup_votor( scratch_a, 0L ); + ag_block_id_t parent = genesis_block_id(); + ag_vote_t notar = send_block_and_expect_notar( a, 1UL, &parent ); + ag_block_id_t tip = ag_block_id( 1UL, notar.notar.block_hash ); + FD_TEST_NO_MSG( a ); + + ag_hist_t plain; + FD_TEST( !ag_votor_hist_export( a, ULONG_MAX, &plain ) ); + FD_TEST( plain.vote_bound==ULONG_MAX ); + ag_votor_set_vote_bound( a, 5UL ); + ag_hist_t bounded; + FD_TEST( !ag_votor_hist_export( a, ULONG_MAX, &bounded ) ); + FD_TEST( bounded.vote_bound==5UL ); + round_trip( &bounded ); + teardown_votor( a ); + + /* Without a vote bound the adopted notar on slot 1 lets slot 2 vote. */ + ag_votor_t * b = setup_votor( scratch_b, 0L ); + FD_TEST( !ag_votor_hist_adopt( b, &plain ) ); + FD_TEST( ag_votor_get_vote_bound( b )==ULONG_MAX ); + send_block_and_expect_notar( b, 2UL, &tip ); + teardown_votor( b ); + + b = setup_votor( scratch_b, 0L ); + FD_TEST( !ag_votor_hist_adopt( b, &bounded ) ); + FD_TEST( ag_votor_get_vote_bound( b )==5UL ); + ag_block_info_t block = {0}; + random_hash( block.hash ); + block.parent = tip; + ag_votor_process_replay( b, 2UL, &block ); + FD_TEST_NO_MSG( b ); + + /* A lower vote bound from a later history does not lower ours. */ + bounded.vote_bound = 3UL; + ag_votor_hist_adopt( b, &bounded ); + FD_TEST( ag_votor_get_vote_bound( b )==5UL ); + teardown_votor( b ); + + FD_LOG_NOTICE(( "pass: bound_travels" )); +} + +static void +expect_adopt_full( ag_votor_t * votor, + ag_hist_t const * hist ) { + round_trip( hist ); + ulong footprint = ag_votor_footprint( TEST_SLOT_MAX ); + fd_memcpy( scratch_b, scratch_a, footprint ); + FD_TEST( ag_votor_hist_adopt( votor, hist )==ULONG_MAX ); + FD_TEST( fd_memeq( scratch_a, scratch_b, footprint ) ); +} + +static void +test_adopt_capacity( void ) { + ag_votor_t * votor = setup_votor( scratch_a, 0L ); + for( ulong slot=1UL; slotslot_states->pool ) ); + + ag_hist_t hist = { .anchor = 12UL, .last_leader_slot = 80UL, .vote_bound = 83UL, .rec_cnt = AG_HIST_MAX }; + for( ulong i=0UL; iroot==4UL && votor->highest_final_cert_slot==12UL ); + FD_TEST( ag_votor_get_vote_bound( votor )==83UL ); + FD_TEST( !slot_state_pool_free( votor->slot_states->pool ) ); + FD_TEST( !ag_votor_hist_adopt( votor, &hist ) ); + + hist.rec_cnt++; + expect_adopt_full( votor, &hist ); + advance_root( votor, 16UL ); + FD_TEST( !ag_votor_hist_adopt( votor, &hist ) ); + FD_TEST( votor->root==8UL && votor->highest_final_cert_slot==16UL ); + FD_TEST( slot_state_pool_free( votor->slot_states->pool )==3UL ); + FD_TEST( !slot_state_map_ele_query_const( votor->slot_states->map, &hist.rec[ 0 ].slot, NULL, votor->slot_states->pool ) ); + FD_TEST( has_voted( votor, 20UL ) ); + + teardown_votor( votor ); + FD_LOG_NOTICE(( "pass: adopt_capacity" )); +} + +static void +test_adopt_equal_anchor_prunes( void ) { + ag_votor_t * votor = setup_votor_at_root( scratch_a, 0L, 100UL ); + + /* A rejected import must leave the pruning cursor and all state + unchanged, even when its records start below the initial root. */ + ag_hist_t hist = { .anchor = 100UL, .last_leader_slot = ULONG_MAX, .vote_bound = ULONG_MAX, .rec_cnt = TEST_SLOT_MAX+1UL }; + for( ulong i=0UL; iroot==104UL && votor->highest_final_cert_slot==112UL ); + for( ulong slot=92UL; slot<=100UL; slot++ ) { + FD_TEST( !slot_state_map_ele_query_const( votor->slot_states->map, &slot, NULL, votor->slot_states->pool ) ); + } + FD_TEST( slot_state_pool_free( votor->slot_states->pool )==TEST_SLOT_MAX-AG_SLOTS_PER_WINDOW ); + + teardown_votor( votor ); + FD_LOG_NOTICE(( "pass: adopt_equal_anchor_prunes" )); +} + +int +main( int argc, + char ** argv ) { + fd_boot( &argc, &argv ); + + test_export_shape(); + test_bound_travels(); + test_adopt_capacity(); + test_adopt_equal_anchor_prunes(); + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} diff --git a/src/disco/events/generated/fd_event_gen.h b/src/disco/events/generated/fd_event_gen.h index 3eaefd8d29d..683297c8a9e 100644 --- a/src/disco/events/generated/fd_event_gen.h +++ b/src/disco/events/generated/fd_event_gen.h @@ -517,6 +517,8 @@ typedef struct fd_event_snapshot_created fd_event_snapshot_created_t; #define FD_EVENT_ADMIN_COMMAND_TYPE_ADD_AUTHORIZED_VOTER (3) /* Add an authorized voter key. */ #define FD_EVENT_ADMIN_COMMAND_TYPE_REMOVE_ALL_AUTHORIZED_VOTERS (4) /* Remove every authorized voter key. */ #define FD_EVENT_ADMIN_COMMAND_TYPE_SNAPSHOT_CREATE (5) /* Request creation of a snapshot. */ +#define FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE (6) /* Take over the identity specified by the given keypair from the machine currently running it. */ +#define FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_STATUS (7) /* Read the running validator's failover state. */ /* The result of processing the admin command. */ #define FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS (1) /* The command completed successfully. */ diff --git a/src/disco/events/schema/admin_command.json b/src/disco/events/schema/admin_command.json index 3f478e037e4..4c0225fcdab 100644 --- a/src/disco/events/schema/admin_command.json +++ b/src/disco/events/schema/admin_command.json @@ -11,7 +11,9 @@ "get_identity": { "description": "Read the running validator identity." }, "add_authorized_voter": { "description": "Add an authorized voter key." }, "remove_all_authorized_voters": { "description": "Remove every authorized voter key." }, - "snapshot_create": { "description": "Request creation of a snapshot." } + "snapshot_create": { "description": "Request creation of a snapshot." }, + "failover_promote": { "description": "Take over the identity specified by the given keypair from the machine currently running it." }, + "failover_status": { "description": "Read the running validator's failover state." } } }, "result": { diff --git a/src/disco/events/schema/events.proto b/src/disco/events/schema/events.proto index ae243783384..1009ee3950d 100644 --- a/src/disco/events/schema/events.proto +++ b/src/disco/events/schema/events.proto @@ -405,6 +405,8 @@ enum AdminCommandType { ADMIN_COMMAND_TYPE_ADD_AUTHORIZED_VOTER = 3; // Add an authorized voter key. ADMIN_COMMAND_TYPE_REMOVE_ALL_AUTHORIZED_VOTERS = 4; // Remove every authorized voter key. ADMIN_COMMAND_TYPE_SNAPSHOT_CREATE = 5; // Request creation of a snapshot. + ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE = 6; // Take over the identity specified by the given keypair from the machine currently running it. + ADMIN_COMMAND_TYPE_FAILOVER_STATUS = 7; // Read the running validator's failover state. } // The result of processing the admin command. diff --git a/src/disco/keyguard/fd_keyguard.h b/src/disco/keyguard/fd_keyguard.h index b852636d91f..f10eef3dab6 100644 --- a/src/disco/keyguard/fd_keyguard.h +++ b/src/disco/keyguard/fd_keyguard.h @@ -36,7 +36,14 @@ FD_STATIC_ASSERT( FD_KEYGUARD_AUTH_VOTERS_MAX<=16UL, auth_voters_max_fits_sign_r #define FD_KEYGUARD_ROLE_RSERVE (8) /* Repair server tile */ #define FD_KEYGUARD_ROLE_VOTOR (9) /* Alpenglow votor tile (QUIC TLS, vote history file) */ #define FD_KEYGUARD_ROLE_TOWER (10) /* Tower tile (tower file) */ -#define FD_KEYGUARD_ROLE_CNT (11) /* number of known roles */ +#define FD_KEYGUARD_ROLE_FAILOV (11) /* Failover admission proof and TLS */ +#define FD_KEYGUARD_ROLE_CNT (12) /* number of known roles */ + +/* Admission proofs use the prepared failover key. */ +#define FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN "FD_FAILOVER_CLI1" +#define FD_KEYGUARD_FAILOVER_SERVER_DOMAIN "FD_FAILOVER_SRV1" +#define FD_KEYGUARD_FAILOVER_DOMAIN_SZ (16UL) +#define FD_KEYGUARD_FAILOVER_PROOF_SZ (FD_KEYGUARD_FAILOVER_DOMAIN_SZ+32UL+32UL) /* Payload types ******************************************************/ @@ -54,6 +61,7 @@ FD_STATIC_ASSERT( FD_KEYGUARD_AUTH_VOTERS_MAX<=16UL, auth_voters_max_fits_sign_r #define FD_KEYGUARD_PAYLOAD_LG_BLS_PUBKEY (12) /* Alpenglow BLS public key query */ #define FD_KEYGUARD_PAYLOAD_LG_TOWER (13) /* Tower file */ #define FD_KEYGUARD_PAYLOAD_LG_VOTE_HISTORY (14) /* Alpenglow vote history file */ +#define FD_KEYGUARD_PAYLOAD_LG_FAILOV (15) /* Failover admission proof */ #define FD_KEYGUARD_PAYLOAD_TXN (1UL< #include @@ -8,6 +9,17 @@ #include #include #include +#include + +static FD_TL uchar junk_public_key[ 32 ]; +static FD_TL int junk_key_fd = -1; + +void +fd_keyload_set_failover_junk( uchar const public_key[ 32 ], + int key_fd ) { + fd_memcpy( junk_public_key, public_key, 32UL ); + junk_key_fd = key_fd; +} uchar * FD_FN_SENSITIVE fd_keyload_read( int key_fd, @@ -79,6 +91,32 @@ read_key( char const * key_path, return fd_keyload_read( key_fd, key_path, key ); } +static void FD_FN_SENSITIVE +read_junk_key( uchar * key ) { + int key_fd = junk_key_fd; + junk_key_fd = -1; + struct stat st; + if( FD_UNLIKELY( fstat( key_fd, &st ) || !S_ISFIFO( st.st_mode ) ) ) + FD_LOG_ERR(( "the failover junk key requires a startup pipe" )); + for( ulong off=0UL; off<64UL; ) { + long n = read( key_fd, key+off, 64UL-off ); + if( FD_UNLIKELY( n==-1L && errno==EINTR ) ) continue; + if( FD_UNLIKELY( n<=0L ) ) FD_LOG_ERR(( "could not read the failover junk key" )); + off += (ulong)n; + } + long n; + do n = read( key_fd, key+64UL, 1UL ); while( FD_UNLIKELY( n==-1L && errno==EINTR ) ); + if( FD_UNLIKELY( n ) ) FD_LOG_ERR(( "unexpected data after the failover junk key" )); + if( FD_UNLIKELY( close( key_fd ) ) ) FD_LOG_ERR(( "closing the failover junk key pipe failed (%i-%s)", errno, fd_io_strerror( errno ) )); + fd_sha512_t * sha = fd_sha512_join( fd_sha512_new( key+FD_SHA512_ALIGN ) ); + FD_TEST( sha ); + fd_ed25519_public_from_private( key+64UL, key, sha ); + if( FD_UNLIKELY( !fd_memeq( key+32UL, key+64UL, 32UL ) || + !fd_memeq( key+32UL, junk_public_key, 32UL ) ) ) + FD_LOG_ERR(( "the failover junk key does not match its public key" )); + fd_memzero_explicit( key+64UL, 4096UL-64UL ); +} + uchar const * FD_FN_SENSITIVE fd_keyload_load( char const * key_path, int public_key_only ) { @@ -91,7 +129,13 @@ fd_keyload_load( char const * key_path, pages. */ uchar * key_page = fd_keyload_alloc_protected_pages( 1UL, 2UL ); - read_key( key_path, key_page ); + if( FD_UNLIKELY( !strcmp( key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) ) { + if( FD_UNLIKELY( fd_memeq( junk_public_key, (uchar[ 32 ]){0}, 32UL ) ) ) FD_LOG_ERR(( "missing failover junk key" )); + if( public_key_only ) fd_memcpy( key_page+32UL, junk_public_key, 32UL ); + else read_junk_key( key_page ); + } else { + read_key( key_path, key_page ); + } if( public_key_only ) fd_memzero_explicit( key_page, 32UL ); @@ -103,6 +147,17 @@ fd_keyload_load( char const * key_path, else return key_page; } +uchar const * FD_FN_SENSITIVE +fd_keyload_generate( void ) { + uchar * key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); + if( FD_UNLIKELY( !fd_rng_secure( key, 32UL ) ) ) FD_LOG_ERR(( "could not generate a keypair" )); + fd_sha512_t * sha = fd_sha512_join( fd_sha512_new( key+FD_SHA512_ALIGN ) ); + FD_TEST( sha ); + fd_ed25519_public_from_private( key+32UL, key, sha ); + fd_memzero_explicit( key+FD_SHA512_ALIGN, sizeof(fd_sha512_t) ); + return fd_keyload_mprotect_ro( key, 0 ); +} + uchar * FD_FN_SENSITIVE fd_keyload_mprotect_wr( uchar const * key, int public_key_only ) { diff --git a/src/disco/keyguard/fd_keyload.h b/src/disco/keyguard/fd_keyload.h index aef02816acc..aa634827041 100644 --- a/src/disco/keyguard/fd_keyload.h +++ b/src/disco/keyguard/fd_keyload.h @@ -3,6 +3,9 @@ #include "../fd_disco_base.h" +/* Internal identity source selected by failover for standby nodes. */ +#define FD_KEYLOAD_FAILOVER_JUNK "@failover-junk" + /* fd_keyload_read() reads a JSON encoded keypair from the provided file descriptor. The key_path is not opened or read from, it is only used to output diagnostic error messages if reading the key fails. @@ -30,7 +33,9 @@ fd_keyload_read( int key_fd, key_path must point to the first letter in a NUL-terminated cstr that is the path on disk of the key file. The key file must exist, be readable, and have the form of a Solana keypair (64 element JSON - array of bytes). If public_key_only is non-zero, zeros out the + array of bytes). FD_KEYLOAD_FAILOVER_JUNK instead selects the + failover junk key installed by fd_keyload_set_failover_junk. + If public_key_only is non-zero, zeros out the private part of the key and returns a pointer to the first byte (of 32) of the public part of the key in binary format. If public_key_only is zero, returns a pointer to the first byte (of 64) @@ -51,6 +56,21 @@ uchar const * FD_FN_SENSITIVE fd_keyload_load( char const * key_path, int public_key_only ); +/* Set this tile thread's failover junk public key and private startup + pipe before loading FD_KEYLOAD_FAILOVER_JUNK. Only Sign gets a pipe, + other tiles pass -1 and can load only the public key. */ + +void +fd_keyload_set_failover_junk( uchar const public_key[ 32 ], + int key_fd ); + +/* Generate a fresh Ed25519 keypair in protected memory. The returned + read-only key has the same unload contract as fd_keyload_load with + public_key_only clear. No key material is written to a file. */ + +uchar const * FD_FN_SENSITIVE +fd_keyload_generate( void ); + /* fd_keyload_mprotect_wr() makes a key page loaded with fd_keyload_load writable. Accepts the const pointer returned by fd_keyload_load and returns a non-const pointer that the caller may diff --git a/src/disco/keyguard/fd_keyswitch.h b/src/disco/keyguard/fd_keyswitch.h index fa3adc2a867..d332605249c 100644 --- a/src/disco/keyguard/fd_keyswitch.h +++ b/src/disco/keyguard/fd_keyswitch.h @@ -20,6 +20,16 @@ /* Application-specific param values should be defined below. */ +/* Sign tile identity switches. KEYPAIR passes a new keypair in + bytes[0..64), PUBKEY passes a public key in bytes[0..32) and selects + the junk key or the failover identity the sign tile already holds. + CHECK passes a public key in bytes[0..32) and checks whether it can + become the failover identity without changing any keys. */ +#define FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR (0UL) +#define FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY (1UL) +#define FD_KEYSWITCH_PARAM_IDENTITY_CHECK (2UL) +#define FD_KEYSWITCH_PARAM_IDENTITY_PREPARE (3UL) /* preload a failover key without installing it */ + #define FD_KEYSWITCH_PARAM_AV_ADD (0UL) #define FD_KEYSWITCH_PARAM_AV_CLEAR (1UL) @@ -29,6 +39,7 @@ struct __attribute__((aligned(FD_KEYSWITCH_ALIGN))) fd_keyswitch_private { ulong result; ulong param; uchar bytes[ 32768UL ]; + ulong failover; /* 1 when the failover tile made this switch, 0 for set-identity */ /* Padding to FD_KEYSWITCH_ALIGN here */ }; diff --git a/src/disco/keyguard/fd_sign_tile.c b/src/disco/keyguard/fd_sign_tile.c index f8b65a0f5dc..dfdf9306726 100644 --- a/src/disco/keyguard/fd_sign_tile.c +++ b/src/disco/keyguard/fd_sign_tile.c @@ -63,6 +63,11 @@ typedef struct { uchar * public_key; uchar * private_key; + uchar const * failover_junk_key; /* read only copies loaded at boot under failover */ + uchar const * failover_boot_key; + uchar * failover_target_key; /* a target key other than the junk and boot keys, zero while there is none */ + uchar const * failover_key; /* the failover target, either failover_boot_key or failover_target_key */ + fd_keyguard_bls_key_t * bls_keys; uchar tip_payment_program [32]; @@ -102,9 +107,90 @@ derive_fields( fd_sign_ctx_t * ctx ) { fd_keyguard_bls_key_derive( &ctx->bls_keys[ 0 ], ctx->public_key, ctx->private_key, ctx->sha512 ); } +/* A failover switch passes only a public key, we copy the junk key or + the failover identity into the identity key. */ + +static void FD_FN_SENSITIVE +failover_select( fd_sign_ctx_t * ctx ) { + uchar const * selected = NULL; + if( FD_LIKELY( ctx->failover_key ) ) { + if( fd_memeq( ctx->keyswitch->bytes, ctx->failover_junk_key+32UL, 32UL ) ) selected = ctx->failover_junk_key; + else if( fd_memeq( ctx->keyswitch->bytes, ctx->failover_key+32UL, 32UL ) ) selected = ctx->failover_key; + } + if( FD_UNLIKELY( !selected ) ) FD_LOG_ERR(( "identity switch selects a key that is neither the junk key nor the failover identity" )); + fd_memzero_explicit( ctx->keyswitch->bytes, 64UL ); + fd_memcpy( ctx->private_key, selected, 32UL ); + fd_memcpy( ctx->public_key, selected+32UL, 32UL ); + derive_fields( ctx ); + fd_keyswitch_state( ctx->keyswitch, FD_KEYSWITCH_STATE_COMPLETED ); +} + +static int +failover_identity_allowed( fd_sign_ctx_t const * ctx, + uchar const * pubkey ) { + /* Selecting junk keeps the existing failover identity. */ + if( fd_memeq( pubkey, ctx->failover_junk_key+32UL, 32UL ) ) return 1; + for( ulong i=0UL; iauthorized_voters_cnt; i++ ) { + if( fd_memeq( pubkey, ctx->authorized_voter_pubkeys[ i ], 32UL ) ) return 0; + } + return 1; +} + +/* failover_keyswitch runs the keyswitches the failover way before + the normal keyswitch code sees them. A failover switch selects the junk key + or the failover target. A prepare loads a new target without installing + it. set-identity also makes the new key the target, unless it is our junk + key. The boot key keeps its read only page from boot, another key is + copied into its own, and the normal switch installs it. Return 0 only + when the normal switch must install this request. Once acknowledged, the + keyswitch belongs to Admin and must not be read again in this + housekeeping call. */ + +static int FD_FN_SENSITIVE +failover_keyswitch( fd_sign_ctx_t * ctx ) { + if( FD_LIKELY( fd_keyswitch_state_query( ctx->keyswitch )!=FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) return 1; + ulong param = fd_keyswitch_param_query( ctx->keyswitch ); + if( FD_UNLIKELY( param==FD_KEYSWITCH_PARAM_IDENTITY_PREPARE ) ) { + int allowed = failover_identity_allowed( ctx, ctx->keyswitch->bytes+32UL ) && + !fd_memeq( ctx->keyswitch->bytes+32UL, ctx->failover_junk_key+32UL, 32UL ); + if( FD_LIKELY( allowed ) ) { + fd_memcpy( ctx->failover_target_key, ctx->keyswitch->bytes, 64UL ); + ctx->failover_key = ctx->failover_target_key; + } + fd_memzero_explicit( ctx->keyswitch->bytes, 64UL ); + ctx->keyswitch->result = allowed ? FD_ADMINCTL_RESULT_SUCCESS : FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER; + fd_keyswitch_state( ctx->keyswitch, allowed ? FD_KEYSWITCH_STATE_COMPLETED : FD_KEYSWITCH_STATE_FAILED ); + return 1; + } + if( FD_UNLIKELY( param==FD_KEYSWITCH_PARAM_IDENTITY_CHECK ) ) { + int allowed = failover_identity_allowed( ctx, ctx->keyswitch->bytes ); + fd_memzero_explicit( ctx->keyswitch->bytes, 64UL ); + ctx->keyswitch->result = allowed ? FD_ADMINCTL_RESULT_SUCCESS : FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER; + fd_keyswitch_state( ctx->keyswitch, allowed ? FD_KEYSWITCH_STATE_COMPLETED : FD_KEYSWITCH_STATE_FAILED ); + return 1; + } + if( FD_UNLIKELY( param==FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY ) ) { + failover_select( ctx ); + return 1; + } + FD_TEST( param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR ); + if( FD_UNLIKELY( !fd_memeq( ctx->keyswitch->bytes+32UL, ctx->failover_junk_key+32UL, 32UL ) ) ) { + if( fd_memeq( ctx->keyswitch->bytes+32UL, ctx->failover_boot_key+32UL, 32UL ) ) { + ctx->failover_key = ctx->failover_boot_key; + fd_memzero_explicit( ctx->failover_target_key, 64UL ); + } else { + fd_memcpy( ctx->failover_target_key, ctx->keyswitch->bytes, 64UL ); + ctx->failover_key = ctx->failover_target_key; + } + } + return 0; +} + static void FD_FN_SENSITIVE during_housekeeping_sensitive( fd_sign_ctx_t * ctx ) { - if( FD_UNLIKELY( fd_keyswitch_state_query( ctx->keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) { + int identity_handled = ctx->failover_junk_key && failover_keyswitch( ctx ); + + if( FD_UNLIKELY( !identity_handled && fd_keyswitch_state_query( ctx->keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) { memcpy( ctx->private_key, ctx->keyswitch->bytes, 32UL ); fd_memzero_explicit( ctx->keyswitch->bytes, 32UL ); FD_COMPILER_MFENCE(); @@ -208,6 +294,25 @@ during_frag( void * _ctx, during_frag_sensitive( _ctx, in_idx, seq, sig, chunk, sz ); } +/* The junk key signs TLS. */ + +static void FD_FN_SENSITIVE +failover_sign( fd_sign_ctx_t * ctx, + uchar * dst, + ulong sz ) { + ulong payload = fd_keyguard_payload_match( ctx->_data, sz, FD_KEYGUARD_SIGN_TYPE_ED25519 ); + if( payload==FD_KEYGUARD_PAYLOAD_TLS_CV ) { + fd_ed25519_sign( dst, ctx->_data, sz, ctx->failover_junk_key+32UL, ctx->failover_junk_key, ctx->sha512 ); + return; + } + if( FD_UNLIKELY( payload!=FD_KEYGUARD_PAYLOAD_FAILOV ) ) FD_LOG_EMERG(( "invalid failover signing domain" )); + if( FD_UNLIKELY( !fd_memeq( ctx->_data+FD_KEYGUARD_FAILOVER_DOMAIN_SZ, ctx->failover_junk_key+32UL, 32UL ) ) ) { + FD_LOG_EMERG(( "admission proof request is not for our junk key" )); + } + if( FD_UNLIKELY( !ctx->failover_key ) ) FD_LOG_EMERG(( "admission proof requested before preparing a failover identity" )); + fd_ed25519_sign( dst, ctx->_data, sz, ctx->failover_key+32UL, ctx->failover_key, ctx->sha512 ); +} + static void FD_FN_SENSITIVE after_frag_sensitive( void * _ctx, ulong in_idx, @@ -261,6 +366,7 @@ after_frag_sensitive( void * _ctx, switch( sign_type ) { case FD_KEYGUARD_SIGN_TYPE_ED25519: { + if( FD_UNLIKELY( role==FD_KEYGUARD_ROLE_FAILOV ) ) { failover_sign( ctx, dst, sz ); break; } fd_ed25519_sign( dst, ctx->_data, sz, ctx->public_key, ctx->private_key, ctx->sha512 ); if( needs_second_sign ) { ulong authority_idx = (sig >> 33) & 0xFUL; @@ -337,13 +443,35 @@ after_frag( void * _ctx, after_frag_sensitive( _ctx, in_idx, seq, sig, sz, tsorig, tspub, stem ); } +/* failover_load_keys sets the failover keys, all NULL unless failover + is enabled. */ + static void FD_FN_SENSITIVE -privileged_init_sensitive( fd_topo_t const * topo, - fd_topo_tile_t const * tile ) { - void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id ); - FD_SCRATCH_ALLOC_INIT( l, scratch ); - fd_sign_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_sign_ctx_t ), sizeof( fd_sign_ctx_t ) ); +failover_load_keys( fd_sign_ctx_t * ctx, + fd_topo_tile_t const * tile ) { + ctx->failover_junk_key = NULL; + ctx->failover_boot_key = NULL; + ctx->failover_target_key = NULL; + ctx->failover_key = NULL; + if( FD_LIKELY( !tile->sign.failover_enabled ) ) return; + + ctx->failover_junk_key = fd_keyload_load( FD_KEYLOAD_FAILOVER_JUNK, /* pubkey only: */ 0 ); + uchar * boot_key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); + fd_memcpy( boot_key, ctx->private_key, 64UL ); + ctx->failover_boot_key = fd_keyload_mprotect_ro( boot_key, /* public_key_only: */ 0 ); + ctx->failover_target_key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); + ctx->failover_key = ctx->failover_boot_key; + if( FD_UNLIKELY( fd_memeq( ctx->failover_junk_key+32UL, ctx->failover_boot_key+32UL, 32UL ) ) ) { + FD_LOG_ERR(( "the failover junk key must differ from the [paths.identity_key] key" )); + } + if( FD_UNLIKELY( tile->sign.failover_boot_to_standby ) ) { + fd_memcpy( ctx->private_key, ctx->failover_junk_key, 64UL ); + } +} +static void FD_FN_SENSITIVE +load_keys( fd_sign_ctx_t * ctx, + fd_topo_tile_t const * tile ) { uchar * identity_key = fd_keyload_mprotect_wr( fd_keyload_load( tile->sign.identity_key_path, /* pubkey only: */ 0 ), /* public_key_only: */ 0 ); ctx->private_key = identity_key; ctx->public_key = identity_key + 32UL; @@ -358,6 +486,18 @@ privileged_init_sensitive( fd_topo_t const * topo, memcpy( ctx->authorized_voter_pubkeys[ i ], authorized_voter_key + 32UL, 32UL ); } + failover_load_keys( ctx, tile ); +} + +static void FD_FN_SENSITIVE +privileged_init_sensitive( fd_topo_t const * topo, + fd_topo_tile_t const * tile ) { + void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id ); + FD_SCRATCH_ALLOC_INIT( l, scratch ); + fd_sign_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_sign_ctx_t ), sizeof( fd_sign_ctx_t ) ); + + load_keys( ctx, tile ); + /* The stack can be taken over and reorganized by under AddressSanitizer, which causes this code to fail. */ #if FD_HAS_ASAN @@ -472,6 +612,12 @@ unprivileged_init_sensitive( fd_topo_t const * topo, FD_TEST( !strcmp( out_link->name, "sign_votor" ) ); FD_TEST( in_link->mtu==FD_KEYGUARD_SIGN_REQ_MTU ); FD_TEST( out_link->mtu==FD_BLS_SIG_SZ ); + } else if( !strcmp(in_link->name, "failov_sign" ) ) { + ctx->in[ i ].role = FD_KEYGUARD_ROLE_FAILOV; + FD_TEST( !strcmp( out_link->name, "sign_failov" ) ); + FD_TEST( in_link->mtu==130UL ); + FD_TEST( out_link->mtu==64UL ); + FD_TEST( ctx->failover_junk_key ); } else { FD_LOG_CRIT(( "unexpected link %s", in_link->name )); } diff --git a/src/disco/keyguard/fuzz_keyguard.c b/src/disco/keyguard/fuzz_keyguard.c index f2ca7f0869c..814591a9967 100644 --- a/src/disco/keyguard/fuzz_keyguard.c +++ b/src/disco/keyguard/fuzz_keyguard.c @@ -1,4 +1,5 @@ #include "fd_keyguard.h" +#include "../../waltz/tls/fd_tls.h" #include @@ -29,5 +30,18 @@ LLVMFuzzerTestOneInput( uchar const * data, (void)fd_keyguard_payload_authorize( &authority, data, size, role, (int)i ); } } + + /* The sign tile checks the failover signing domain, so the failov role + signs only the admission proof or a CertificateVerify. */ + for( ulong i=0UL; i #include @@ -295,6 +296,269 @@ test_bls_request_rejected( void ) { FD_TEST( bls_request_exit_status( FD_KEYGUARD_SIGN_TYPE_BLS|(1UL<<32), 11UL )==1 ); /* authorized voter 0, none loaded */ } +/* With failover, the sign tile switches by public key between the junk + key and the failover identity, the staked key from boot or a key + set-identity installed. */ + +static uchar junk_key [ 64 ]; +static uchar staked_key [ 64 ]; +static uchar operator_key[ 64 ]; /* the sign tile's page for a set-identity key */ + +static void +failover_setup( void ) { + setup(); + fd_memset( junk_key, 0x33, 32UL ); + fd_memset( staked_key, 0x44, 32UL ); + fd_ed25519_public_from_private( junk_key +32UL, junk_key, ctx.sha512 ); + fd_ed25519_public_from_private( staked_key+32UL, staked_key, ctx.sha512 ); + fd_memcpy( identity_key, junk_key, 64UL ); /* a failover member boots under the junk key */ + derive_fields( &ctx ); + fd_memset( operator_key, 0, 64UL ); + ctx.failover_junk_key = junk_key; + ctx.failover_boot_key = staked_key; + ctx.failover_target_key = operator_key; + ctx.failover_key = staked_key; + ctx.in[0].role = FD_KEYGUARD_ROLE_FAILOV; + client.response_mtu = FD_ED25519_SIG_SZ; +} + +static void * +sign_one( void * arg ) { + ulong seq = *(ulong *)arg; + fd_frag_meta_t const * line = client.request+fd_mcache_line_idx( seq, TEST_DEPTH ); + while( fd_frag_meta_seq_query( line )!=seq ) FD_SPIN_PAUSE(); + FD_COMPILER_MFENCE(); + during_frag( &ctx, 0UL, seq, line->sig, line->chunk, line->sz, 0UL ); + after_frag( &ctx, 0UL, seq, line->sig, line->sz, 0UL, 0UL, &stem ); + return NULL; +} + +/* Whether the sign tile signs payload with the key of public_key. */ + +static int +failover_signed_by( uchar const * payload, + ulong payload_sz, + uchar const * public_key ) { + uchar signature[ FD_ED25519_SIG_SZ ]; + ulong seq = client.request_seq; + pthread_t signer; + FD_TEST( !pthread_create( &signer, NULL, sign_one, &seq ) ); + fd_keyguard_client_sign( &client, signature, payload, payload_sz, FD_KEYGUARD_SIGN_TYPE_ED25519 ); + FD_TEST( !pthread_join( signer, NULL ) ); + return FD_ED25519_SUCCESS==fd_ed25519_verify( payload, payload_sz, signature, public_key, ctx.sha512 ); +} + +static void +failover_switch( fd_keyswitch_t * ks, + uchar const * public_key ) { + fd_memcpy( ks->bytes, public_key, 32UL ); + fd_memset( ks->bytes+32UL, 0, 32UL ); + ks->param = FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY; + fd_keyswitch_state( ks, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( ks )==FD_KEYSWITCH_STATE_COMPLETED ); + for( ulong i=0UL; i<64UL; i++ ) FD_TEST( !ks->bytes[ i ] ); +} + +/* Without failover an identity switch takes the keypair whatever the + param says, the failover key selection does not run. */ + +static void +test_identity_switch_without_failover( void ) { + setup(); + fd_keyswitch_t identity_switch[1]; + ctx.keyswitch = fd_keyswitch_join( fd_keyswitch_new( identity_switch, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.keyswitch ); + uchar keypair[ 64 ]; + fd_memset( keypair, 0x55, 32UL ); + fd_ed25519_public_from_private( keypair+32UL, keypair, ctx.sha512 ); + fd_memcpy( ctx.keyswitch->bytes, keypair, 64UL ); + ctx.keyswitch->param = FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY; + fd_keyswitch_state( ctx.keyswitch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( ctx.keyswitch )==FD_KEYSWITCH_STATE_COMPLETED ); + FD_TEST( fd_memeq( ctx.private_key, keypair, 32UL ) && fd_memeq( ctx.public_key, keypair+32UL, 32UL ) ); + + /* Adding an authorized voter takes the normal path, whatever its + key. */ + fd_keyswitch_t voter_switch[1]; + ctx.av_keyswitch = fd_keyswitch_join( fd_keyswitch_new( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); + FD_TEST( ctx.av_keyswitch ); + fd_memcpy( voter_switch->bytes, keypair, 64UL ); + voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_COMPLETED && ctx.authorized_voters_cnt==1UL ); +} + +static void +test_failover_keys( void ) { + failover_setup(); + fd_keyswitch_t identity_switch[1]; + ctx.keyswitch = fd_keyswitch_join( fd_keyswitch_new( identity_switch, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.keyswitch ); + + uchar proof[ FD_KEYGUARD_FAILOVER_PROOF_SZ ] = {0}; + fd_memcpy( proof, FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN, FD_KEYGUARD_FAILOVER_DOMAIN_SZ ); + fd_memcpy( proof+FD_KEYGUARD_FAILOVER_DOMAIN_SZ, junk_key+32UL, 32UL ); + uchar cv[ FD_TLS_CV_SIGN_SZ ]; + fd_memcpy( cv, fd_tls13_cli_sign_prefix, sizeof(fd_tls13_cli_sign_prefix) ); + fd_memset( cv+sizeof(fd_tls13_cli_sign_prefix), 0x5A, sizeof(cv)-sizeof(fd_tls13_cli_sign_prefix) ); + + fd_keyguard_bls_key_t junk_bls[1], staked_bls[1]; + fd_keyguard_bls_key_derive( junk_bls, junk_key +32UL, junk_key, ctx.sha512 ); + fd_keyguard_bls_key_derive( staked_bls, staked_key+32UL, staked_key, ctx.sha512 ); + + /* Junk, staked, then junk again. Whichever is installed, the staked + key signs the proof and the junk key the handshake, and BLS + key 0 follows the installed identity. */ + for( ulong i=0UL; i<3UL; i++ ) { + uchar const * installed = i==1UL ? staked_key : junk_key; + fd_keyguard_bls_key_t const * bls = i==1UL ? staked_bls : junk_bls; + if( i ) failover_switch( identity_switch, installed+32UL ); + FD_TEST( fd_memeq( ctx.public_key, installed+32UL, 32UL ) ); + FD_TEST( fd_memeq( ctx.private_key, installed, 32UL ) ); + FD_TEST( fd_memeq( keys[0].public_key, bls->public_key, FD_BLS_PUB_COMPRESSED_SZ ) ); + FD_TEST( failover_signed_by( proof, sizeof(proof), staked_key+32UL ) ); + FD_TEST( failover_signed_by( cv, sizeof(cv), junk_key +32UL ) ); + } + + /* The failover identity can become an authorized voter as on main, + admin checks it when failover prepares or switches to it. */ + fd_keyswitch_t voter_switch[1]; + ctx.av_keyswitch = fd_keyswitch_join( fd_keyswitch_new( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); + FD_TEST( ctx.av_keyswitch ); + fd_memcpy( voter_switch->bytes, staked_key, 64UL ); + voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_COMPLETED ); + FD_TEST( ctx.authorized_voters_cnt==1UL && fd_memeq( ctx.authorized_voter_pubkeys[ 0 ], staked_key+32UL, 32UL ) ); + for( ulong i=0UL; i<32UL; i++ ) FD_TEST( !voter_switch->bytes[ i ] ); + FD_LOG_NOTICE(( "pass: failover keys" )); +} + +/* A set-identity keypair other than the junk key becomes the failover + identity, it signs the admission proof and cannot become an + authorized voter. The staked key keeps its boot page, another key goes in the + operator page. */ + +static void +operator_switch( fd_keyswitch_t * ks, + uchar const * keypair ) { + fd_memcpy( ks->bytes, keypair, 64UL ); + ks->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR; + fd_keyswitch_state( ks, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( ks )==FD_KEYSWITCH_STATE_COMPLETED ); +} + +static void +test_failover_target_key( void ) { + failover_setup(); + fd_keyswitch_t identity_switch[1]; + ctx.keyswitch = fd_keyswitch_join( fd_keyswitch_new( identity_switch, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.keyswitch ); + uchar third[ 64 ]; + fd_memset( third, 0x66, 32UL ); + fd_ed25519_public_from_private( third+32UL, third, ctx.sha512 ); + uchar proof[ FD_KEYGUARD_FAILOVER_PROOF_SZ ] = {0}; + fd_memcpy( proof, FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN, FD_KEYGUARD_FAILOVER_DOMAIN_SZ ); + fd_memcpy( proof+FD_KEYGUARD_FAILOVER_DOMAIN_SZ, junk_key+32UL, 32UL ); + + uchar staked_copy[ 64 ]; + fd_memcpy( staked_copy, staked_key, 64UL ); + operator_switch( identity_switch, third ); + FD_TEST( fd_memeq( ctx.public_key, third+32UL, 32UL ) && fd_memeq( operator_key, third, 64UL ) ); + FD_TEST( ctx.failover_key==operator_key && fd_memeq( staked_key, staked_copy, 64UL ) ); + FD_TEST( failover_signed_by( proof, sizeof(proof), third+32UL ) ); + + /* A failover switch can now select the new key and the junk key. */ + failover_switch( identity_switch, junk_key+32UL ); + FD_TEST( fd_memeq( ctx.public_key, junk_key+32UL, 32UL ) ); + failover_switch( identity_switch, third+32UL ); + FD_TEST( fd_memeq( ctx.public_key, third+32UL, 32UL ) ); + + /* set-identity to the junk key keeps the failover identity. */ + operator_switch( identity_switch, junk_key ); + FD_TEST( fd_memeq( ctx.public_key, junk_key+32UL, 32UL ) && ctx.failover_key==operator_key && fd_memeq( operator_key, third, 64UL ) ); + FD_TEST( failover_signed_by( proof, sizeof(proof), third+32UL ) ); + + /* Authorized voters change as on main, also to the failover + identity, admin checks it when failover switches to it. */ + fd_keyswitch_t voter_switch[1]; + ctx.av_keyswitch = fd_keyswitch_join( fd_keyswitch_new( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); + FD_TEST( ctx.av_keyswitch ); + fd_memcpy( voter_switch->bytes, third, 64UL ); + voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_COMPLETED && ctx.authorized_voters_cnt==1UL ); + fd_keyswitch_state( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + fd_memcpy( voter_switch->bytes, staked_key, 64UL ); + voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_COMPLETED && ctx.authorized_voters_cnt==2UL ); + + /* set-identity to the boot key puts it back on its boot page and + clears the operator page. */ + voter_switch->param = FD_KEYSWITCH_PARAM_AV_CLEAR; + fd_keyswitch_state( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( !ctx.authorized_voters_cnt ); + operator_switch( identity_switch, staked_key ); + uchar zero[ 64 ] = {0}; + FD_TEST( ctx.failover_key==staked_key && fd_memeq( operator_key, zero, 64UL ) ); + FD_TEST( failover_signed_by( proof, sizeof(proof), staked_key+32UL ) ); + failover_switch( identity_switch, junk_key+32UL ); + failover_switch( identity_switch, staked_key+32UL ); + FD_TEST( fd_memeq( ctx.public_key, staked_key+32UL, 32UL ) ); + FD_LOG_NOTICE(( "pass: failover operator key" )); +} + +static fd_keyswitch_t failover_ks[1]; + +static void +switch_to_unknown_key( void ) { + uchar other[ 64 ]; + fd_memset( other, 0x55, 32UL ); + fd_ed25519_public_from_private( other+32UL, other, ctx.sha512 ); + failover_switch( failover_ks, other+32UL ); +} + +static void +sign_foreign_proof( void ) { + fd_memcpy( ctx._data, FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN, FD_KEYGUARD_FAILOVER_DOMAIN_SZ ); + fd_memcpy( ctx._data+FD_KEYGUARD_FAILOVER_DOMAIN_SZ, staked_key+32UL, 32UL ); + after_frag( &ctx, 0UL, 0UL, FD_KEYGUARD_SIGN_TYPE_ED25519, FD_KEYGUARD_FAILOVER_PROOF_SZ, 0UL, 0UL, &stem ); +} + +static int +child_exit_status( void (*fn)( void ) ) { + pid_t pid = fork(); + FD_TEST( pid>=0 ); + if( !pid ) { + fd_log_level_core_set( 8 ); + fn(); + _exit( 0 ); + } + int status; + FD_TEST( waitpid( pid, &status, 0 )==pid ); + FD_TEST( WIFEXITED( status ) ); + return WEXITSTATUS( status ); +} + +/* A switch to a key that is neither the junk key nor the failover + identity (as well as a proof over another junk key) all stop the + validator. */ + +static void +test_failover_refusals( void ) { + failover_setup(); + ctx.keyswitch = fd_keyswitch_join( fd_keyswitch_new( failover_ks, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.keyswitch ); + FD_TEST( child_exit_status( switch_to_unknown_key )==1 ); + FD_TEST( child_exit_status( sign_foreign_proof )==1 ); + FD_LOG_NOTICE(( "pass: failover refusals" )); +} + int main( int argc, char ** argv ) { @@ -309,6 +573,10 @@ main( int argc, test_bls_pubkey_last_authority(); test_bls_pubkey_request_rejected(); test_bls_request_rejected(); + test_identity_switch_without_failover(); + test_failover_keys(); + test_failover_target_key(); + test_failover_refusals(); FD_LOG_NOTICE(( "pass" )); fd_halt(); return 0; diff --git a/src/disco/topo/fd_topo.c b/src/disco/topo/fd_topo.c index a4f74c778f2..b1cbd59d098 100644 --- a/src/disco/topo/fd_topo.c +++ b/src/disco/topo/fd_topo.c @@ -222,6 +222,11 @@ fd_topo_tile_extra_huge_pages( fd_topo_tile_t const * tile ) { FD_FN_PURE static ulong fd_topo_tile_extra_normal_pages( fd_topo_tile_t const * tile ) { ulong key_pages = 0UL; + if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) && tile->sign.failover_enabled ) ) { + /* The junk key, the read only copy of the boot key and the failover + target key, each with its guard pages, next to the identity key. */ + key_pages += 15UL; + } if( FD_UNLIKELY( tile->id_keyswitch_obj_id!=ULONG_MAX ) ) { /* Certain tiles using fd_keyload_load need normal pages to hold key material. */ @@ -232,6 +237,11 @@ fd_topo_tile_extra_normal_pages( fd_topo_tile_t const * tile ) { key material. */ key_pages += 5UL; } + if( FD_UNLIKELY( !strcmp( tile->name, "failov" ) ) ) { + /* The failover tile loads one public key at a time at boot and + unloads it before the next. */ + key_pages += 5UL; + } if( !strcmp( tile->name, "net" ) ) { /* net tile uses normal pages to hold xsk rings */ diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index 889b8df1021..98ffa75129d 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -245,6 +245,17 @@ struct fd_topo_tile { char identity_key_path[ PATH_MAX ]; } admin; + struct { + char identity_key_path[ PATH_MAX ]; + char vote_account_path[ PATH_MAX ]; + int boot_to_standby; + uint listen_addr; /* network byte order */ + ushort port; + + char gossip_host[ FD_FQDN_BUF_MAX ]; + fd_ip4_port_t gossip_addr; /* our own gossip socket, gossip_host overrides the address */ + } failov; + #define FD_TOPO_GOSSIP_ENTRYPOINTS_MAX 16UL struct { @@ -403,6 +414,9 @@ struct fd_topo_tile { struct { char identity_key_path[ PATH_MAX ]; + int failover_junk_fd; /* private startup pipe for the failover junk key, -1 without failover */ + int failover_enabled; + int failover_boot_to_standby; ulong authorized_voter_paths_cnt; char authorized_voter_paths[ FD_KEYGUARD_AUTH_VOTERS_MAX ][ PATH_MAX ]; struct { @@ -811,6 +825,7 @@ typedef struct { between them. */ struct fd_topo { char app_name[ 256UL ]; + uchar failover_junk[ 32 ]; /* public half of the junk key generated at boot for failover, zero without failover */ uchar props[ 32768UL ]; ulong sleep_obj_id; diff --git a/src/disco/topo/fd_topo_run.c b/src/disco/topo/fd_topo_run.c index 38ee1e711af..365f8b0b606 100644 --- a/src/disco/topo/fd_topo_run.c +++ b/src/disco/topo/fd_topo_run.c @@ -6,6 +6,7 @@ #include "../sleep/fd_sleep.h" #include "../metrics/fd_metrics.h" #include "../events/fd_event_report.h" +#include "../keyguard/fd_keyload.h" #include "../../util/tile/fd_tile_private.h" #include @@ -78,6 +79,8 @@ fd_topo_run_tile( fd_topo_t * topo, fd_topo_join_tile_workspaces( topo, tile, core_dump_level ); fd_topo_mlock_code(); + fd_keyload_set_failover_junk( topo->failover_junk, strcmp( tile->name, "sign" ) ? -1 : tile->sign.failover_junk_fd ); + if( FD_UNLIKELY( tile_run->privileged_init ) ) tile_run->privileged_init( topo, tile ); @@ -232,7 +235,9 @@ fd_topo_tile_stack_join( char const * app_name, char name[ PATH_MAX ]; FD_TEST( fd_cstr_printf_check( name, PATH_MAX, NULL, "%s_stack_%s%lu", app_name, tile_name, tile_kind_id ) ); - int dump = strcmp( tile_name, "sign" ) ? 1 : 0; /* avoid core dumps of sign tile stacks */ + /* avoid core dumps of sign tile stacks, and of failov ones, which hold + TLS session secrets */ + int dump = strcmp( tile_name, "sign" ) && strcmp( tile_name, "failov" ) ? 1 : 0; uchar * stack = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, dump, NULL, NULL, NULL ); if( FD_UNLIKELY( !stack ) ) FD_LOG_ERR(( "fd_shmem_join failed" )); diff --git a/src/disco/topo/fd_topob_layout.c b/src/disco/topo/fd_topob_layout.c index a80d6251ee0..80d727b72b1 100644 --- a/src/disco/topo/fd_topob_layout.c +++ b/src/disco/topo/fd_topob_layout.c @@ -72,6 +72,7 @@ static struct { char const * name; uint flags; } const TILE_CLASS[] = { { "genesi", FLOATING }, { "ipecho", FLOATING }, { "admin", FLOATING }, + { "failov", FLOATING }, { "snapsv", FLOATING|PIN_EFF }, { "snapmk", FLOATING }, { "snapzp", FLOATING }, diff --git a/src/discof/admin/fd_admin_tile.c b/src/discof/admin/fd_admin_tile.c index 6fefacc7a53..e72589ccd9d 100644 --- a/src/discof/admin/fd_admin_tile.c +++ b/src/discof/admin/fd_admin_tile.c @@ -4,6 +4,7 @@ #include "../../disco/keyguard/fd_keyload.h" #include "fd_adminctl.h" +#include "../failover/fd_failover_bus.h" #include #include "generated/fd_admin_tile_seccomp.h" @@ -24,6 +25,37 @@ struct fd_admin_tile_ctx { long poll_ticks; long idle_poll_ticks; long deadline_ticks; /* next adminctl poll */ + + int failover_enabled; /* failover hands off the identity */ + int failover_switch; /* a failover identity switch is in progress */ + ulong failover_epoch; /* failover epoch, bumped by every set-identity under failover */ + int failover_pending; /* the failover tile has not been told about the last set-identity yet */ + uchar junk_pubkey[ 32 ]; /* the junk key generated at boot, a completed handover switches to it */ + uchar failover_pubkey[ 32 ]; /* the key failover hands off, the boot identity until set-identity or a prepare picks another */ + + /* Failover commands go to the failover tile over the bus, one at a + time like snapshot creation. */ + ulong failov_out_idx; /* admin_failov stem out index */ + fd_wksp_t * failov_out_mem; + ulong failov_out_chunk0; + ulong failov_out_wmark; + ulong failov_out_chunk; + ulong failov_in_idx; /* failov_admin stem in index */ + ulong in_cnt; /* stem ins, failov_admin is found among them */ + fd_wksp_t * failov_in_mem; + ulong failov_in_chunk0; + ulong failov_in_wmark; + fd_failover_bus_msg_t failov_in; /* frame copied in during_frag */ + ulong failover_slot_idx; /* adminctl slot parked on the bus, or ULONG_MAX */ + ulong failover_slot_cmd; /* FD_ADMINCTL_FAILOVER_CMD_* */ + ulong failover_nonce; /* nonce of the last command forwarded to the failover tile */ + ulong failover_answered; /* nonce of the last command the failover tile responded to */ + ulong failover_start_time; /* command start retained until the failover tile responds */ + long failover_deadline; /* tickcount after which we report unresponsive */ + char failover_args_json[ 96 ]; /* the parked command and its flags for the event */ + ulong failover_args_json_len; + char failover_cmd_cstr[ 48 ]; /* the parked command and its flags for the log, empty for status */ + int failover_retargeted; /* the parked command changed the failover target before it was forwarded */ }; typedef struct fd_admin_tile_ctx fd_admin_tile_ctx_t; @@ -88,6 +120,22 @@ privileged_init( fd_topo_t const * topo, FD_LOG_ERR(( "identity_key_path not set" )); fd_memcpy( ctx->identity_pubkey, fd_keyload_load( tile->admin.identity_key_path, /* pubkey only: */ 1 ), 32UL ); + + /* With failover, the configured identity is the key that failover + hands off, and a handoff switches to the junk key generated at boot. */ + ulong failov_idx = fd_topo_find_tile( topo, "failov", 0UL ); + if( FD_UNLIKELY( failov_idx!=ULONG_MAX ) ) { + FD_TEST( !fd_memeq( topo->failover_junk, (uchar[ 32 ]){0}, 32UL ) ); + fd_memcpy( ctx->junk_pubkey, topo->failover_junk, 32UL ); + fd_topo_tile_t const * failov = &topo->tiles[ failov_idx ]; + if( FD_UNLIKELY( failov->failov.boot_to_standby ) ) { + uchar const * target = fd_keyload_load( failov->failov.identity_key_path, /* pubkey only: */ 1 ); + fd_memcpy( ctx->failover_pubkey, target, 32UL ); + fd_keyload_unload( target, 1 ); + } else { + fd_memcpy( ctx->failover_pubkey, ctx->identity_pubkey, 32UL ); + } + } } static void @@ -100,7 +148,9 @@ unprivileged_init( fd_topo_t const * topo, ctx->poll_ticks = (long)(1e6*fd_tempo_tick_per_ns( NULL )); /* 1ms */ ctx->idle_poll_ticks = (long)((double)FD_SLEEP_PARK_CAP_NS*fd_tempo_tick_per_ns( NULL )); ctx->deadline_ticks = 0L; - ctx->topo = topo; + ctx->failover_enabled = fd_topo_find_tile( topo, "failov", 0UL )!=ULONG_MAX; + ctx->failover_slot_idx = ULONG_MAX; + ctx->topo = topo; fd_topo_obj_t const * adminctl_obj = fd_topo_find_tile_obj( topo, tile, "adminctl" ); FD_TEST( adminctl_obj ); @@ -110,8 +160,33 @@ unprivileged_init( fd_topo_t const * topo, ctx->replay_out_idx = fd_topo_find_tile_out_link( topo, tile, "admin_replay", 0UL ); + /* The failover bus links exist only when the failover tile does. */ + ctx->failov_out_idx = fd_topo_find_tile_out_link( topo, tile, "admin_failov", 0UL ); + ctx->failov_in_idx = fd_topo_find_tile_in_link ( topo, tile, "failov_admin", 0UL ); + ctx->in_cnt = tile->in_cnt; + if( FD_UNLIKELY( (ctx->failov_out_idx==ULONG_MAX)!=(ctx->failov_in_idx==ULONG_MAX) ) ) { + FD_LOG_ERR(( "admin tile needs both the admin_failov and failov_admin links or neither" )); + } + if( FD_UNLIKELY( ctx->failov_out_idx!=ULONG_MAX ) ) { + fd_topo_link_t const * out_link = &topo->links[ tile->out_link_id[ ctx->failov_out_idx ] ]; + ctx->failov_out_mem = topo->workspaces[ topo->objs[ out_link->dcache_obj_id ].wksp_id ].wksp; + ctx->failov_out_chunk0 = fd_dcache_compact_chunk0( ctx->failov_out_mem, out_link->dcache ); + ctx->failov_out_wmark = fd_dcache_compact_wmark ( ctx->failov_out_mem, out_link->dcache, out_link->mtu ); + ctx->failov_out_chunk = ctx->failov_out_chunk0; + + fd_topo_link_t const * in_link = &topo->links[ tile->in_link_id[ ctx->failov_in_idx ] ]; + ctx->failov_in_mem = topo->workspaces[ topo->objs[ in_link->dcache_obj_id ].wksp_id ].wksp; + ctx->failov_in_chunk0 = fd_dcache_compact_chunk0( ctx->failov_in_mem, in_link->dcache ); + ctx->failov_in_wmark = fd_dcache_compact_wmark ( ctx->failov_in_mem, in_link->dcache, in_link->mtu ); + } + for( ulong i=0UL; iin_cnt; i++ ) { fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ]; + /* The stem numbers only the in links it polls, and we compare its + numbers with topology in link indexes, so with failover every in + link is polled. */ + if( FD_UNLIKELY( ctx->failov_in_idx!=ULONG_MAX ) ) FD_TEST( tile->in_link_poll[ i ] ); + if( FD_UNLIKELY( i==ctx->failov_in_idx ) ) continue; if( FD_UNLIKELY( strcmp( link->name, "replay_admin" ) ) ) { FD_LOG_ERR(( "unexpected input link name %s", link->name )); } @@ -421,6 +496,13 @@ poll_set_identity( fd_admin_tile_ctx_t * ctx, if( strcmp( tile->name, "sign" ) ) continue; fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id ); memcpy( sign->bytes, keypair, 64UL ); + if( FD_UNLIKELY( ctx->failover_switch ) ) { + /* With failover, the sign tile selects the junk key or the + failover identity by its public key. */ + sign->param = FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY; + fd_memcpy( sign->bytes, keypair+32UL, 32UL ); + fd_memset( sign->bytes+32UL, 0, 32UL ); + } FD_COMPILER_MFENCE(); sign->state = FD_KEYSWITCH_STATE_SWITCH_PENDING; FD_COMPILER_MFENCE(); @@ -566,8 +648,101 @@ poll_set_identity( fd_admin_tile_ctx_t * ctx, return *state==FD_SET_IDENTITY_STATE_UNLOCKED; } +/* Check every sign tile before changing the failover identity or halting + any tiles. Admin serializes commands, so authorized voters cannot + change between this check and the identity switch. */ + +static void +failover_identity_check_begin( fd_admin_tile_ctx_t * ctx, + uchar const * identity ) { + fd_topo_t const * topo = ctx->topo; + for( ulong i=0UL; itile_cnt; i++ ) { + fd_topo_tile_t const * tile = &topo->tiles[ i ]; + if( strcmp( tile->name, "sign" ) ) continue; + fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id ); + fd_memcpy( sign->bytes, identity, 32UL ); + sign->param = FD_KEYSWITCH_PARAM_IDENTITY_CHECK; + fd_keyswitch_state( sign, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + } +} + +/* Returns ULONG_MAX until every sign tile has answered. */ +static ulong +failover_identity_check_poll( fd_admin_tile_ctx_t * ctx ) { + fd_topo_t const * topo = ctx->topo; + ulong result = FD_ADMINCTL_RESULT_SUCCESS; + for( ulong i=0UL; itile_cnt; i++ ) { + fd_topo_tile_t const * tile = &topo->tiles[ i ]; + if( strcmp( tile->name, "sign" ) ) continue; + fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id ); + ulong state = fd_keyswitch_state_query( sign ); + if( state==FD_KEYSWITCH_STATE_SWITCH_PENDING ) return ULONG_MAX; + FD_TEST( state==FD_KEYSWITCH_STATE_COMPLETED || state==FD_KEYSWITCH_STATE_FAILED ); + if( state==FD_KEYSWITCH_STATE_FAILED ) result = sign->result; + } + for( ulong i=0UL; itile_cnt; i++ ) { + fd_topo_tile_t const * tile = &topo->tiles[ i ]; + if( strcmp( tile->name, "sign" ) ) continue; + fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id ); + sign->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR; + fd_keyswitch_state( sign, FD_KEYSWITCH_STATE_UNLOCKED ); + } + return result; +} + +/* Runs until every sign tile has responded. */ +static ulong +failover_identity_check( fd_admin_tile_ctx_t * ctx, + uchar const * identity ) { + failover_identity_check_begin( ctx, identity ); + ulong result; + while( (result = failover_identity_check_poll( ctx ))==ULONG_MAX ) FD_SPIN_PAUSE(); + return result; +} + +/* With failover, set-identity still switches the same way. */ + +static void +failover_operator_begin( fd_admin_tile_ctx_t * ctx, + uchar const * identity ) { + ctx->failover_epoch++; + if( !fd_memeq( identity, ctx->junk_pubkey, 32UL ) ) fd_memcpy( ctx->failover_pubkey, identity, 32UL ); + FD_BASE58_ENCODE_32_BYTES( identity, new_identity ); + FD_BASE58_ENCODE_32_BYTES( ctx->failover_pubkey, failover_identity ); + FD_LOG_NOTICE(( "set-identity switches to `%s`, failover restarts on this machine for identity `%s`", new_identity, failover_identity )); + + fd_topo_t const * topo = ctx->topo; + for( ulong i=0UL; itile_cnt; i++ ) { + if( strcmp( topo->tiles[ i ].name, "sign" ) ) continue; + fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, topo->tiles[ i ].id_keyswitch_obj_id ); + sign->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR; + } + find_identity_keyswitch( ctx, "votor" )->failover = 0UL; + ctx->failover_pending = 1; +} + +/* Sends a pending OPERATOR once the failover link has room to spare. */ + +static void +failover_operator_notify( fd_admin_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + if( FD_LIKELY( !ctx->failover_pending ) ) return; + if( FD_UNLIKELY( ctx->failov_out_idx==ULONG_MAX || stem->cr_avail[ ctx->failov_out_idx ]<4UL ) ) return; + fd_failover_bus_msg_t * msg = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk ); + fd_memset( msg, 0, sizeof(*msg) ); + fd_failover_operator_t operator = { .epoch=ctx->failover_epoch }; + fd_memcpy( operator.identity, ctx->identity_pubkey, 32UL ); + fd_memcpy( operator.failover_identity, ctx->failover_pubkey, 32UL ); + fd_memcpy( msg->payload, &operator, sizeof(operator) ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_OPERATOR_ACTION, ctx->failov_out_chunk, sizeof(*msg), 0UL, tspub, tspub ); + ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*msg), ctx->failov_out_chunk0, ctx->failov_out_wmark ); + ctx->failover_pending = 0; +} + static void FD_FN_SENSITIVE set_identity( fd_admin_tile_ctx_t * ctx, + fd_stem_context_t * stem, ulong slot_idx, void * data, ulong data_sz ) { @@ -617,6 +792,8 @@ set_identity( fd_admin_tile_ctx_t * ctx, return; } + if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_operator_begin( ctx, req->keypair+32UL ); + ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; ulong identity_outset = (ulong)fd_log_wallclock(); for(;;) { @@ -624,6 +801,7 @@ set_identity( fd_admin_tile_ctx_t * ctx, } memcpy( ctx->identity_pubkey, req->keypair+32UL, 32UL ); + if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_operator_notify( ctx, stem ); report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS ); fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_SUCCESS ); @@ -677,7 +855,7 @@ get_identity( fd_admin_tile_ctx_t * ctx, done carefully in order to prevent votes being generated with an authorized voter that the sign tile is not yet aware of. The authorized voter must be added to the sign tile before it is added - to the voter tile (the tower/votor tile). All transitions must be + to the votor tile. All transitions must be linear and in forward order. */ /* State 0: UNLOCKED @@ -986,8 +1164,7 @@ snapshot_create_response( fd_admin_tile_ctx_t * ctx, /* Removing all authorized voters from the validator is the inverse of add-authorized-voter, and must be done in the opposite order. When - adding, the sign tile is updated before the voter tile (the tower - tile, or the votor tile under Alpenglow) so that the voter never asks + adding, the sign tile is updated before the votor tile so that it never asks the sign tile to sign a vote with an authority index the sign tile does not yet know about. When removing, the voter tile must be cleared before the sign tiles, so that the voter stops referencing an @@ -1168,6 +1345,350 @@ next_deadline( fd_admin_tile_ctx_t const * ctx ) { return ctx->deadline_ticks; } +/* The failover tile responds to commands. We check the + ABI, forward the command over the bus and park the adminctl slot + until the response comes back or the deadline passes, like snapshot + creation waits on replay. */ + +static fd_event_admin_command_t * +failover_event_args( fd_admin_tile_ctx_t const * ctx, + fd_event_admin_command_t * event ) { + if( FD_LIKELY( ctx->failover_args_json_len ) ) { + fd_memcpy( event->args_json, ctx->failover_args_json, ctx->failover_args_json_len ); + event->args_json_len = ctx->failover_args_json_len; + } + return event; +} + +/* failover_result_name is the event's custom result for a failover + command that did not succeed. */ +static char const * +failover_result_name( ulong result ) { + switch( result ) { + case FD_FAILOVER_CONTROL_RESULT_BUSY: return "busy"; + case FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE: return "unresponsive"; + case FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER: return "wrong_holder"; + case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: return "in_progress"; + case FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY: return "peer_unready"; + case FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS: return "peer_holds"; + case FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING: return "handoff_pending"; + case FD_FAILOVER_CONTROL_RESULT_TAKEN: return "taken"; + case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "staked_seen"; + case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: return "no_final_history"; + case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: return "replay_behind"; + case FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS: return "no_holder_address"; + case FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT: return "key_conflict"; + case FD_FAILOVER_CONTROL_RESULT_INPUT_LOST: return "input_lost"; + case FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH: return "keypair_mismatch"; + case FD_SET_IDENTITY_RESULT_SAME_IDENTITY: return "same_identity"; + case FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER: return "authorized_voter"; + default: return "refused"; + } +} + +static void +failover_complete( fd_admin_tile_ctx_t * ctx, + ulong result, + void const * resp, + ulong resp_sz ) { + fd_event_admin_command_t event = { + .type = ctx->failover_slot_cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_STATUS : FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE, + .args_json = { '{', '}' }, + .args_json_len = 2UL, + .start_time = ctx->failover_start_time, + .payload_version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, + .has_payload_version = 1, + .payload_size = sizeof(fd_adminctl_failover_req_t), + }; + switch( result ) { + case FD_ADMINCTL_RESULT_SUCCESS: + report_admin_command( failover_event_args( ctx, &event ), FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS ); + break; + default: + report_admin_command_custom_result( failover_event_args( ctx, &event ), failover_result_name( result ) ); + break; + } + fd_adminctl_complete_response( ctx->adminctl, ctx->failover_slot_idx, result, resp, resp_sz ); + if( FD_LIKELY( ctx->failover_cmd_cstr[ 0 ] ) ) { + if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) { + FD_LOG_NOTICE(( "`%s` accepted by the failover tile, follow it with `failover status`", ctx->failover_cmd_cstr )); + } else if( FD_UNLIKELY( result!=FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE ) ) { + if( FD_UNLIKELY( ctx->failover_retargeted ) ) { + FD_BASE58_ENCODE_32_BYTES( ctx->failover_pubkey, target ); + FD_LOG_WARNING(( "`%s` refused by the failover tile (%s); the failover target is now `%s`, nothing else was done", ctx->failover_cmd_cstr, failover_result_name( result ), target )); + } else { + FD_LOG_WARNING(( "`%s` refused by the failover tile (%s), nothing was done", ctx->failover_cmd_cstr, failover_result_name( result ) )); + } + } + } + ctx->failover_retargeted = 0; + ctx->failover_slot_idx = ULONG_MAX; + ctx->failover_start_time = 0UL; + ctx->failover_deadline = 0L; + ctx->failover_args_json_len = 0UL; + ctx->failover_cmd_cstr[ 0 ] = '\0'; +} + +/* Load a target key without changing the installed identity. */ +static ulong FD_FN_SENSITIVE +failover_prepare( fd_admin_tile_ctx_t * ctx, + uchar const * keypair ) { + uchar pubkey[ 32 ]; + fd_ed25519_public_from_private( pubkey, keypair, ctx->sha512 ); + if( FD_UNLIKELY( !fd_memeq( pubkey, keypair+32UL, 32UL ) ) ) return FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH; + if( FD_UNLIKELY( fd_memeq( pubkey, ctx->junk_pubkey, 32UL ) ) ) return FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT; + if( fd_memeq( pubkey, ctx->failover_pubkey, 32UL ) ) return FD_ADMINCTL_RESULT_SUCCESS; + ulong result = failover_identity_check( ctx, pubkey ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) return result; + + for( ulong i=0UL; itopo->tile_cnt; i++ ) { + fd_topo_tile_t const * tile = &ctx->topo->tiles[ i ]; + if( strcmp( tile->name, "sign" ) ) continue; + fd_keyswitch_t * sign = fd_topo_obj_laddr( ctx->topo, tile->id_keyswitch_obj_id ); + fd_memcpy( sign->bytes, keypair, 64UL ); + sign->param = FD_KEYSWITCH_PARAM_IDENTITY_PREPARE; + fd_keyswitch_state( sign, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + } + while( (result = failover_identity_check_poll( ctx ))==ULONG_MAX ) FD_SPIN_PAUSE(); + /* Preflight checked every sign tile before any target was loaded. */ + FD_TEST( result==FD_ADMINCTL_RESULT_SUCCESS ); + fd_memcpy( ctx->failover_pubkey, pubkey, 32UL ); + ctx->failover_epoch++; + ctx->failover_pending = 1; + ctx->failover_retargeted = 1; + return FD_ADMINCTL_RESULT_SUCCESS; +} + +static void FD_FN_SENSITIVE +failover_request( fd_admin_tile_ctx_t * ctx, + fd_stem_context_t * stem, + ulong slot_idx, + ulong cmd_id, + void * data, + ulong data_sz ) { + + int status = cmd_id==FD_ADMINCTL_CMD_FAILOVER_STATUS; + int event_type = status ? FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_STATUS : FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE; + fd_adminctl_t * adminctl = ctx->adminctl; + fd_event_admin_command_t event = prepare_admin_command( event_type, data, data_sz ); + + if( FD_UNLIKELY( data_sz=FD_ADMINCTL_FAILOVER_CMD_CNT ) invalid = "unknown command"; + else if( (req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS)!=status ) invalid = "payload command does not match the admin command"; + else if( req.flags & ~(FD_ADMINCTL_FAILOVER_FLAG_YES|FD_ADMINCTL_FAILOVER_FLAG_FORCE) ) invalid = "unknown flags"; + else if( (req.cmd==FD_ADMINCTL_FAILOVER_CMD_PROMOTE)!=force ) invalid = "--force goes with promote, and a promote without it is sent as a handoff"; + else if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS && req.flags ) invalid = "failover status takes no flags"; + else if( req.cmd!=FD_ADMINCTL_FAILOVER_CMD_HANDOFF && (req.addr || req.port) ) invalid = "an address or port goes only with a handoff"; + if( FD_UNLIKELY( invalid ) ) { + FD_LOG_WARNING(( "refusing adminctl failover cmd %lu flags %lu, %s", req.cmd, req.flags, invalid )); + report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNKNOWN_COMMAND ); + fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ); + return; + } + if( !status ) { + FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, + "{\"yes\":%s,\"force\":%s}", + (req.flags & FD_ADMINCTL_FAILOVER_FLAG_YES) ? "true" : "false", + force ? "true" : "false" ) ); + } + char cmd_cstr[ 48 ]; + FD_TEST( fd_cstr_printf_check( cmd_cstr, sizeof(cmd_cstr), NULL, "%s%s%s", fd_adminctl_failover_cmd_typed( req.cmd ), + (req.flags & FD_ADMINCTL_FAILOVER_FLAG_YES) ? " --yes" : "", + force ? " --force" : "" ) ); + + if( FD_UNLIKELY( ctx->failov_out_idx==ULONG_MAX ) ) { + FD_LOG_WARNING(( "`%s` refused, the admin tile has no failover bus link", cmd_cstr )); + report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNSUPPORTED ); + fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNSUPPORTED ); + return; + } + + /* Keep one command outstanding until the failover tile responds, even + after timeout. Status reports a stalled tile as unresponsive. Publish + the last set-identity OPERATOR first so the command uses its identity. */ + if( FD_UNLIKELY( ctx->failover_answered!=ctx->failover_nonce || ctx->failover_pending ) ) { + if( FD_UNLIKELY( status && ( ctx->failover_slot_idx==ULONG_MAX || ctx->failover_pending ) ) ) { + FD_LOG_WARNING(( "`failover status` refused, the failover tile has not responded to an earlier command, check the log" )); + report_admin_command_custom_result( &event, "unresponsive" ); + fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE ); + return; + } + FD_LOG_WARNING(( "`%s` refused, another failover command is still waiting on the failover tile, try again", cmd_cstr )); + report_admin_command_custom_result( &event, "busy" ); + fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_BUSY ); + return; + } + + if( prepare ) { + if( FD_UNLIKELY( stem->cr_avail[ ctx->failov_out_idx ]<4UL ) ) { + report_admin_command_custom_result( &event, "busy" ); + fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_BUSY ); + return; + } + fd_adminctl_failover_prepare_t * payload = data; + ctx->failover_retargeted = 0; + ulong result = failover_prepare( ctx, payload->keypair ); + fd_memzero_explicit( payload->keypair, sizeof(payload->keypair) ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + FD_LOG_WARNING(( "`%s` refused before it reached the failover tile (%s), nothing was done", cmd_cstr, failover_result_name( result ) )); + report_admin_command_custom_result( &event, "target_refused" ); + fd_adminctl_complete( adminctl, slot_idx, result ); + return; + } + /* The public target and epoch must reach the controller before the + command. after_credit reserves enough credits for both frames. */ + failover_operator_notify( ctx, stem ); + FD_TEST( !ctx->failover_pending ); + } + + /* Kept for the event and the log of the response. */ + FD_TEST( event.args_json_len<=sizeof(ctx->failover_args_json) ); + fd_memcpy( ctx->failover_args_json, event.args_json, event.args_json_len ); + ctx->failover_args_json_len = event.args_json_len; + if( status ) ctx->failover_cmd_cstr[ 0 ] = '\0'; + else { + fd_cstr_ncpy( ctx->failover_cmd_cstr, cmd_cstr, sizeof(ctx->failover_cmd_cstr) ); + FD_LOG_NOTICE(( "forwarding `%s` to the failover tile", cmd_cstr )); + } + + fd_failover_bus_msg_t * msg = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk ); + fd_memset( msg, 0, sizeof(*msg) ); + msg->nonce = ++ctx->failover_nonce; + fd_memcpy( msg->payload, &req, sizeof(req) ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_REQUEST, ctx->failov_out_chunk, sizeof(*msg), 0UL, tspub, tspub ); + ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*msg), ctx->failov_out_chunk0, ctx->failov_out_wmark ); + + ctx->failover_slot_idx = slot_idx; + ctx->failover_slot_cmd = req.cmd; + ctx->failover_start_time = event.start_time; + ctx->failover_deadline = fd_tickcount() + (long)( (double)FD_FAILOVER_BUS_DEADLINE_NANOS*fd_tempo_tick_per_ns( NULL ) ); +} + +static void +failover_response( fd_admin_tile_ctx_t * ctx ) { + fd_failover_bus_msg_t const * msg = &ctx->failov_in; + /* Even a late response means the last command is no longer outstanding. */ + if( FD_LIKELY( msg->nonce==ctx->failover_nonce ) ) ctx->failover_answered = msg->nonce; + /* A response for an older command, or one after the deadline already + answered the operator, is dropped. */ + if( FD_UNLIKELY( ctx->failover_slot_idx==ULONG_MAX || msg->nonce!=ctx->failover_nonce ) ) { + FD_LOG_WARNING(( "dropping a stale failover command response" )); + return; + } + /* Status and command responses each have their own payload. */ + ulong resp_sz = ctx->failover_slot_cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? sizeof(fd_adminctl_failover_status_resp_t) + : sizeof(fd_adminctl_failover_control_resp_t); + failover_complete( ctx, msg->result, msg->payload, resp_sz ); +} + +/* The failover tile asked for an identity switch. We only pass the + public key on, the sign tile selects the junk key or the failover + identity. + Like set-identity this blocks until every tile has switched. We poll + no adminctl command meanwhile, so `failover status` and every failover + command wait for it. */ + +static void +failover_switch_request( fd_admin_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + ulong nonce = ctx->failov_in.nonce; + fd_failover_switch_req_t req; + fd_memcpy( &req, ctx->failov_in.payload, sizeof(req) ); + + fd_failover_switch_resp_t response; + fd_memset( &response, 0, sizeof(response) ); + response.result = FD_FAILOVER_SWITCH_ERR_DISABLED; + if( FD_UNLIKELY( ctx->failover_enabled && req.epoch!=ctx->failover_epoch ) ) { + FD_LOG_WARNING(( "refusing the failover tile's identity switch, set-identity ran since it asked" )); + response.result = FD_FAILOVER_SWITCH_ERR_STALE; + response.operator.epoch = ctx->failover_epoch; + fd_memcpy( response.operator.identity, ctx->identity_pubkey, 32UL ); + fd_memcpy( response.operator.failover_identity, ctx->failover_pubkey, 32UL ); + } else if( FD_UNLIKELY( ctx->failover_enabled && !fd_memeq( req.identity, ctx->junk_pubkey, 32UL ) && + failover_identity_check( ctx, req.identity )!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + FD_BASE58_ENCODE_32_BYTES( req.identity, target ); + FD_LOG_WARNING(( "refusing the failover tile's identity switch to `%s`, it is one of the authorized voters, remove it with remove-all-authorized-voters first", target )); + response.result = FD_FAILOVER_SWITCH_ERR_AUTHORIZED_VOTER; + } else if( FD_LIKELY( ctx->failover_enabled ) ) { + fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_SET_IDENTITY, NULL, 0UL ); + FD_BASE58_ENCODE_32_BYTES( ctx->identity_pubkey, old_identity ); + FD_BASE58_ENCODE_32_BYTES( req.identity, new_identity ); + FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, + "{\"old_identity\":\"%s\",\"identity\":\"%s\",\"source\":\"failover\"}", old_identity, new_identity ) ); + FD_LOG_NOTICE(( "failover: switching the identity from `%s` to `%s`, failover commands wait until every tile has switched", old_identity, new_identity )); + + /* No private half, only the public key reaches the keyswitches. */ + uchar keypair[ 64 ]; + fd_memset( keypair, 0, 32UL ); + fd_memcpy( keypair+32UL, req.identity, 32UL ); + + find_identity_keyswitch( ctx, "votor" )->failover = 1UL; /* votor enforces adoption before target participation */ + ctx->failover_switch = 1; + ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; + ulong identity_outset = (ulong)fd_log_wallclock(); + for(;;) { + if( FD_UNLIKELY( poll_set_identity( ctx, &state, identity_outset, keypair, 0UL ) ) ) break; + } + ctx->failover_switch = 0; + fd_memcpy( ctx->identity_pubkey, req.identity, 32UL ); + + /* Votor returns its final votor_hist sequence after halting signing. */ + response.result = FD_FAILOVER_SWITCH_OK; + response.history_watermark = find_identity_keyswitch( ctx, "votor" )->result; + report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS ); + FD_LOG_NOTICE(( "failover: every tile switched to `%s`, votor halted signing at sequence %lu", new_identity, response.history_watermark )); + } else { + FD_LOG_WARNING(( "the failover tile asked for an identity switch but failover is off, refusing" )); + } + + fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk ); + fd_memset( out, 0, sizeof(*out) ); + out->nonce = nonce; + fd_memcpy( out->payload, &response, sizeof(response) ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_SWITCH_RESP, ctx->failov_out_chunk, sizeof(*out), 0UL, tspub, tspub ); + ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*out), ctx->failov_out_chunk0, ctx->failov_out_wmark ); +} + +/* Returns whether a frag is waiting on failov_admin. */ + +static int +failov_frag_waiting( fd_admin_tile_ctx_t const * ctx, + fd_stem_context_t const * stem ) { + for( ulong i=0UL; iin_cnt; i++ ) { + fd_stem_tile_in_t const * in = &stem->in[ i ]; + if( FD_LIKELY( (ulong)in->idx==ctx->failov_in_idx ) ) return fd_seq_diff( in->seq, fd_frag_meta_seq_query( in->mline ) )<=0L; + } + return 0; +} + static inline void FD_FN_SENSITIVE after_credit( fd_admin_tile_ctx_t * ctx, fd_stem_context_t * stem, @@ -1179,8 +1700,18 @@ after_credit( fd_admin_tile_ctx_t * ctx, void * payload = NULL; ulong payload_sz = 0UL; + failover_operator_notify( ctx, stem ); + + if( FD_UNLIKELY( ctx->failover_slot_idx!=ULONG_MAX && fd_tickcount()>ctx->failover_deadline && + !failov_frag_waiting( ctx, stem ) ) ) { + if( FD_LIKELY( ctx->failover_cmd_cstr[ 0 ] ) ) FD_LOG_WARNING(( "the failover tile did not answer `%s` in time, it may still run, check `failover status` and the log", ctx->failover_cmd_cstr )); + else FD_LOG_WARNING(( "the failover tile did not answer `failover status` in time, check the log" )); + failover_complete( ctx, FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE, NULL, 0UL ); + *charge_busy = 1; + } + if( FD_LIKELY( stem->nowdeadline_ticks ) ) return; - ctx->deadline_ticks = stem->now + fd_long_if( ctx->snap_create_slot_idx!=ULONG_MAX, ctx->poll_ticks, ctx->idle_poll_ticks ); + ctx->deadline_ticks = stem->now + fd_long_if( ( ctx->snap_create_slot_idx!=ULONG_MAX || ctx->failover_slot_idx!=ULONG_MAX ), ctx->poll_ticks, ctx->idle_poll_ticks ); ulong cmd_id = FD_ADMINCTL_CMD_IDLE; for( ulong i=0UL; ifailover_enabled ) ) *opt_poll_in = 0; /* it may have published OPERATOR */ break; case FD_ADMINCTL_CMD_REMOVE_ALL_AUTH_VOTERS: remove_all_authorized_voters( ctx, slot_idx, payload, payload_sz ); @@ -1209,6 +1741,19 @@ after_credit( fd_admin_tile_ctx_t * ctx, *charge_busy = 1; *opt_poll_in = 0; break; + case FD_ADMINCTL_CMD_FAILOVER_PROMOTE: + case FD_ADMINCTL_CMD_FAILOVER_STATUS: + if( FD_UNLIKELY( ctx->failov_out_idx==ULONG_MAX ) ) { + /* The command reads UNKNOWN_COMMAND as failover disabled. */ + FD_LOG_NOTICE(( "`%s` requested but failover is not enabled on this validator", + cmd_id==FD_ADMINCTL_CMD_FAILOVER_STATUS ? "failover status" : "failover promote" )); + fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ); + break; + } + failover_request( ctx, stem, slot_idx, cmd_id, payload, payload_sz ); + *charge_busy = 1; + *opt_poll_in = 0; + break; default: FD_LOG_WARNING(( "unexpected adminctl cmd %lu", cmd_id )); fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ); @@ -1223,6 +1768,14 @@ during_frag( fd_admin_tile_ctx_t * ctx, ulong chunk FD_PARAM_UNUSED, ulong sz FD_PARAM_UNUSED, ulong ctl ) { + if( FD_UNLIKELY( in_idx==ctx->failov_in_idx ) ) { + if( FD_UNLIKELY( chunkfailov_in_chunk0 || chunk>ctx->failov_in_wmark || sz!=sizeof(fd_failover_bus_msg_t) ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->failov_in_chunk0, ctx->failov_in_wmark )); + } + fd_memcpy( &ctx->failov_in, fd_chunk_to_laddr_const( ctx->failov_in_mem, chunk ), sizeof(fd_failover_bus_msg_t) ); + return; + } + if( FD_UNLIKELY( ctx->snap_create_slot_idx==ULONG_MAX ) ) { FD_LOG_ERR(( "unexpected replay snapshot-create response with no pending adminctl command" )); return; @@ -1230,6 +1783,29 @@ during_frag( fd_admin_tile_ctx_t * ctx, snapshot_create_response( ctx, sig, ctl ); } +static void +after_frag( fd_admin_tile_ctx_t * ctx, + ulong in_idx, + ulong seq FD_PARAM_UNUSED, + ulong sig, + ulong sz FD_PARAM_UNUSED, + ulong tsorig FD_PARAM_UNUSED, + ulong tspub FD_PARAM_UNUSED, + fd_stem_context_t * stem ) { + if( FD_LIKELY( in_idx!=ctx->failov_in_idx ) ) return; + switch( sig ) { + case FD_FAILOVER_BUS_SWITCH_REQ: + failover_switch_request( ctx, stem ); + break; + case FD_FAILOVER_BUS_RESPONSE: + failover_response( ctx ); + break; + default: + FD_LOG_WARNING(( "unexpected failover bus frame %lu", sig )); + break; + } +} + static ulong populate_allowed_seccomp( fd_topo_t const * topo FD_PARAM_UNUSED, fd_topo_tile_t const * tile FD_PARAM_UNUSED, @@ -1255,7 +1831,7 @@ populate_allowed_fds( fd_topo_t const * topo FD_PARAM_UNUSED, return out_cnt; } -#define STEM_BURST (1UL) +#define STEM_BURST (2UL) #define STEM_LAZY ((long)1e6) /* 1ms */ #define STEM_ALWAYS_PARK 1 @@ -1265,6 +1841,7 @@ populate_allowed_fds( fd_topo_t const * topo FD_PARAM_UNUSED, #define STEM_CALLBACK_NEXT_DEADLINE next_deadline #define STEM_CALLBACK_AFTER_CREDIT after_credit #define STEM_CALLBACK_DURING_FRAG during_frag +#define STEM_CALLBACK_AFTER_FRAG after_frag #include "../../disco/stem/fd_stem.c" diff --git a/src/discof/admin/fd_adminctl.h b/src/discof/admin/fd_adminctl.h index f57c3e24ccc..ae95028aa13 100644 --- a/src/discof/admin/fd_adminctl.h +++ b/src/discof/admin/fd_adminctl.h @@ -37,6 +37,8 @@ #define FD_ADMINCTL_CMD_GET_IDENTITY (3UL) #define FD_ADMINCTL_CMD_REMOVE_ALL_AUTH_VOTERS (4UL) #define FD_ADMINCTL_CMD_SNAP_CREATE (5UL) +#define FD_ADMINCTL_CMD_FAILOVER_PROMOTE (6UL) +#define FD_ADMINCTL_CMD_FAILOVER_STATUS (7UL) #define FD_ADMINCTL_ALIGN (8UL) #define FD_ADMINCTL_PAYLOAD_MAX (32768UL) @@ -63,6 +65,22 @@ #define FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH (0x3001UL) #define FD_SET_IDENTITY_RESULT_INVALID_VOTE_HISTORY (0x3002UL) #define FD_SET_IDENTITY_RESULT_SAME_IDENTITY (0x3003UL) +#define FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER (0x3004UL) + +#define FD_FAILOVER_CONTROL_RESULT_BUSY (0x4001UL) /* another failover command is waiting on the failover tile */ +#define FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE (0x4002UL) /* the failover tile did not respond in time */ +#define FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER (0x4003UL) /* this machine already runs the identity, or the one it asked does not */ +#define FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS (0x4004UL) /* a transition or key switch is running */ +#define FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY (0x4005UL) /* the bound peer cannot complete this handoff */ +#define FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS (0x4006UL) /* the authenticated peer runs the identity */ +#define FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING (0x4007UL) /* the peer has not responded to our handoff */ +#define FD_FAILOVER_CONTROL_RESULT_TAKEN (0x4008UL) /* the peer took our handoff, cleared once we hold the identity again or set-identity restarts failover */ +#define FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN (0x4009UL) /* gossip has a fresh contact info for the staked identity from another host */ +#define FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY (0x400AUL) /* the machine running the identity has no eligible final vote state, see its local voting diagnostics */ +#define FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS (0x400BUL) /* gossip has no address for the machine running the identity */ +#define FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND (0x400CUL) /* this machine's replay is too far behind the last vote of the machine running the identity */ +#define FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT (0x400DUL) /* the keypair is the junk key */ +#define FD_FAILOVER_CONTROL_RESULT_INPUT_LOST (0x400EUL) /* failover input overrun, restart required */ struct fd_adminctl_add_auth_voter_v1 { ulong version; /* ==FD_ADMINCTL_ADD_AUTH_VOTER_PAYLOAD_VERSION */ @@ -107,6 +125,122 @@ struct fd_adminctl_remove_all_auth_voters_v1 { typedef struct fd_adminctl_remove_all_auth_voters_v1 fd_adminctl_remove_all_auth_voters_t; #define FD_ADMINCTL_REMOVE_ALL_AUTH_VOTERS_PAYLOAD_VERSION (1UL) +/* Failover commands, forwarded to the failover tile. */ +#define FD_ADMINCTL_FAILOVER_CMD_HANDOFF (0UL) +#define FD_ADMINCTL_FAILOVER_CMD_PROMOTE (1UL) +#define FD_ADMINCTL_FAILOVER_CMD_STATUS (2UL) +#define FD_ADMINCTL_FAILOVER_CMD_CNT (3UL) + +/* The command as the operator typed it. */ +static inline char const * +fd_adminctl_failover_cmd_typed( ulong cmd ) { + switch( cmd ) { + case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: + case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: return "failover promote"; + case FD_ADMINCTL_FAILOVER_CMD_STATUS: return "failover status"; + default: return "unknown"; + } +} + +#define FD_ADMINCTL_FAILOVER_FLAG_YES (1UL) /* confirmation only, does not authorize peer or history overrides */ +#define FD_ADMINCTL_FAILOVER_FLAG_FORCE (2UL) /* --force, bypass peer guards and accept incomplete or empty vote history */ + +struct fd_adminctl_failover_req_v1 { + ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ + ulong cmd; /* FD_ADMINCTL_FAILOVER_CMD_* */ + ulong flags; /* FD_ADMINCTL_FAILOVER_FLAG_* */ + uint addr; /* HANDOFF only, the machine running the identity to dial in network byte order, 0 to find it in gossip */ + ushort port; /* HANDOFF only, the port to dial it on in host byte order, 0 for the default port */ +}; +typedef struct fd_adminctl_failover_req_v1 fd_adminctl_failover_req_t; + +/* Optional promote payload. Admin consumes and wipes the keypair. + Only req may be forwarded to the failover tile. */ +struct fd_adminctl_failover_prepare_v1 { + fd_adminctl_failover_req_t req; + uchar keypair[ 64 ]; +}; +typedef struct fd_adminctl_failover_prepare_v1 fd_adminctl_failover_prepare_t; + +struct fd_adminctl_failover_control_resp_v1 { + ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ + uchar holds; /* this machine runs the identity after the command */ + uchar action; /* controller action after the command */ + ulong handoff_id; /* the handoff an accepted promote opened or resumed, 0 none */ +}; +typedef struct fd_adminctl_failover_control_resp_v1 fd_adminctl_failover_control_resp_t; +#define FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION (1UL) + +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_req_t )==32UL, failover_req_v1_layout ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)==24UL, failover_control_resp_v1_layout ); + +/* What the failover controller is doing right now. */ +#define FD_FAILOVER_ACTION_IDLE (0UL) /* no local transition is running */ +#define FD_FAILOVER_ACTION_DEMOTE_SWITCH (1UL) /* waiting for the junk key to be installed */ +#define FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK (2UL) /* DEMOTED sent, waiting for the peer */ +#define FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY (3UL) /* waiting for replay to reach the finality anchor */ +#define FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT (4UL) /* waiting for votor to adopt */ +#define FD_FAILOVER_ACTION_PROMOTE_SWITCH (5UL) /* waiting for the staked key to be installed */ +#define FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER (6UL) /* requesting the final history from the machine running the identity */ +#define FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT (7UL) /* waiting for the machine that handed it over to record our response */ +#define FD_FAILOVER_ACTION_DEMOTE_DRAIN (8UL) /* junk key installed, waiting for votor history to drain */ +#define FD_FAILOVER_ACTION_CNT (9UL) + +/* The history sources a promotion can adopt, in order of preference. */ +#define FD_FAILOVER_SOURCE_PEER (0UL) /* the history the peer's DEMOTED gave us */ +#define FD_FAILOVER_SOURCE_EMPTY (1UL) /* forced empty history with a vote bound */ +#define FD_FAILOVER_SOURCE_CNT (2UL) + +/* The outcome of the last handoff this machine requested or gave. */ +#define FD_FAILOVER_HANDOFF_NONE (0UL) /* nothing requested or sent */ +#define FD_FAILOVER_HANDOFF_PENDING (1UL) /* requested or sent, no answer yet */ +#define FD_FAILOVER_HANDOFF_TAKEN (2UL) /* the requesting machine took the identity */ +#define FD_FAILOVER_HANDOFF_DECLINED (3UL) /* it did not take the identity, or the machine that handed it over did not record that it did */ +#define FD_FAILOVER_HANDOFF_RESTARTED (4UL) /* the peer came back with a new boot_id */ +#define FD_FAILOVER_HANDOFF_CANCELLED (5UL) /* `failover promote --force` stopped waiting for the peer */ +#define FD_FAILOVER_HANDOFF_NOT_HOLDER (6UL) /* the machine we dialed does not run the identity */ +#define FD_FAILOVER_HANDOFF_CNT (7UL) + +/* `failover status`, provided by the failover tile. */ +struct fd_adminctl_failover_status_resp_v1 { + ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ + uchar enabled; /* 1 when failover runs */ + uchar holds; /* this machine runs the identity */ + uchar action; /* FD_FAILOVER_ACTION_* */ + uchar stuck; /* a transition failed or is overdue */ + uchar link_state; /* FD_FAILOVER_SESSION_* */ + uchar peer_holds_valid; /* a peer is authenticated on the current session */ + uchar peer_holds; /* the peer runs the identity, from HELLO or the handoff result */ + uchar handoff_result; /* FD_FAILOVER_HANDOFF_* of our last handoff */ + ulong peer_boot_id; /* boot_id of the last peer we paired with, 0 none */ + uint peer_addr; /* the address we dial, from `failover promote --address` or from gossip, 0 none */ + ushort peer_port; /* the failover port we dial it on, in host byte order */ + uchar promote_source; /* FD_FAILOVER_SOURCE_* `failover promote --force` would adopt now */ + uchar peer_addr_cmd; /* peer_addr is from `failover promote --address`, else it came from gossip */ + ulong handoff_id; /* id of our last handoff, 0 none */ + ulong promote_floor; /* coverage floor a promote has to reach, ULONG_MAX none */ + ulong promote_result; /* what the takeover checks report without --force now, SUCCESS when they find no other holder */ + uchar mode; /* FD_FAILOVER_MODE_* the validator runs */ + uchar request_paused; /* our handoff request stopped at its deadline, `failover promote` resumes it */ + uchar target[ 32 ]; /* the identity failover hands off, from [paths.identity_key], set-identity or `failover promote ` */ + uchar installed[ 32 ]; /* the identity this machine signs with now */ + uchar junk[ 32 ]; /* this boot's junk key, installed when the identity is handed off */ +}; +typedef struct fd_adminctl_failover_status_resp_v1 fd_adminctl_failover_status_resp_t; + +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_status_resp_t)==160UL, failover_status_resp_v1_layout ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_status_resp_t)<=FD_ADMINCTL_PAYLOAD_MAX, failover_status_resp_fits ); + +/* fd_adminctl_failover_status_resp_init clears a status response and + sets its version and the unknown promote floor. */ + +static inline void +fd_adminctl_failover_status_resp_init( fd_adminctl_failover_status_resp_t * resp ) { + fd_memset( resp, 0, sizeof(*resp) ); + resp->version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION; + resp->promote_floor = ULONG_MAX; +} + typedef struct fd_adminctl_private fd_adminctl_t; FD_PROTOTYPES_BEGIN diff --git a/src/discof/admin/test_admin_tile.c b/src/discof/admin/test_admin_tile.c index f1a59788c37..86e50a423f3 100644 --- a/src/discof/admin/test_admin_tile.c +++ b/src/discof/admin/test_admin_tile.c @@ -193,6 +193,210 @@ test_set_identity( void ) { FD_TEST( state==FD_SET_IDENTITY_STATE_UNLOCKED && replay_ks->state==FD_KEYSWITCH_STATE_UNLOCKED ); } +static uchar adminctl_mem[ 1UL<<18 ] __attribute__((aligned(FD_ADMINCTL_ALIGN))); + +/* With failover, set-identity changes the failover identity unless the + key is our junk key. OPERATOR waits while the link has no room and + failover commands are busy until it goes out. A switch request from + before set-identity gets STALE with the failover identity. */ + +static fd_frag_meta_t bus_mcache[ 8 ]; +static ulong bus_seq; +static ulong bus_depth = 8UL; +static ulong bus_cr_avail; +static ulong bus_min_cr_avail; +static int bus_reliable = 1; +static fd_frag_meta_t * bus_mcaches[ 1 ] = { bus_mcache }; +static fd_stem_context_t bus_stem[ 1 ]; +static uchar bus_out[ 1024 ] __attribute__((aligned(128))); + +static void +bus_setup( ulong credits ) { + fd_memset( bus_mcache, 0, sizeof(bus_mcache) ); + bus_seq = 0UL; + bus_cr_avail = credits; + bus_min_cr_avail = credits; + *bus_stem = (fd_stem_context_t){ .mcaches=bus_mcaches, .seqs=&bus_seq, .depths=&bus_depth, + .cr_avail=&bus_cr_avail, .min_cr_avail=&bus_min_cr_avail, + .cr_decrement_amount=1UL, .out_reliable=&bus_reliable }; + ctx.failov_out_idx = 0UL; + ctx.failov_out_mem = (fd_wksp_t *)bus_out; + ctx.failov_out_chunk0 = ctx.failov_out_wmark = ctx.failov_out_chunk = 0UL; +} + +static fd_failover_operator_t +bus_operator( void ) { + FD_TEST( bus_seq && bus_mcache[ bus_seq-1UL ].sig==FD_FAILOVER_BUS_OPERATOR_ACTION ); + fd_failover_operator_t operator; + fd_memcpy( &operator, ((fd_failover_bus_msg_t const *)bus_out)->payload, sizeof(operator) ); + return operator; +} + +/* Events identify the public command on completion and on rejection, + including requests whose payload cannot be decoded. */ +static void +test_failover_command( ulong cmd_id, + ulong payload_cmd, + ulong flags, + int event_type ) { + setup(); + bus_setup( 8UL ); + ctx.adminctl = fd_adminctl_join( fd_adminctl_new( adminctl_mem ) ); + ctx.failover_enabled = 1; + ctx.failover_slot_idx = ULONG_MAX; + FD_TEST( ctx.adminctl ); + + static fd_frag_meta_t event_mcache[ 128 ] __attribute__((aligned(128))); + static uchar event_mem[ 1024 ] __attribute__((aligned(128))); + ulong event_seq = 0UL; + fd_event_reporter_t reporter = { .mcache=event_mcache, .depth=128UL, .seq_store=&event_seq, + .mem=(fd_wksp_t *)event_mem, .mtu=sizeof(event_mem) }; + fd_event_tl = &reporter; + + for( ulong i=0UL; i<4UL; i++ ) { + void * payload; + ulong payload_max; + ulong slot_idx = fd_adminctl_reserve( ctx.adminctl, &payload, &payload_max ); + FD_TEST( slot_idx!=ULONG_MAX && payload_max>=sizeof(fd_adminctl_failover_req_t) ); + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=payload_cmd, .flags=flags }; + /* A status command must not carry a promotion, or vice versa. */ + if( i==3UL ) req.cmd = payload_cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? FD_ADMINCTL_FAILOVER_CMD_HANDOFF : FD_ADMINCTL_FAILOVER_CMD_STATUS; + fd_memcpy( payload, &req, sizeof(req) ); + fd_adminctl_publish( ctx.adminctl, slot_idx, cmd_id, i==2UL ? 0UL : sizeof(req) ); + int poll_in = 1; + int busy = 0; + ulong seq_before = bus_seq; + for( ulong j=0UL; jpayload, sizeof(forwarded) ); + FD_TEST( forwarded.cmd==payload_cmd && forwarded.flags==flags ); + result = i ? FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER : FD_ADMINCTL_RESULT_SUCCESS; + event_result = i ? FD_EVENT_ADMIN_COMMAND_RESULT_CUSTOM : FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS; + ctx.failov_in.nonce = ctx.failover_nonce; + ctx.failov_in.result = result; + failover_response( &ctx ); + } else { + FD_TEST( bus_seq==seq_before ); + result = i==2UL ? FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH : FD_ADMINCTL_RESULT_UNKNOWN_COMMAND; + event_result = i==2UL ? FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH : FD_EVENT_ADMIN_COMMAND_RESULT_UNKNOWN_COMMAND; + } + FD_TEST( fd_adminctl_wait( ctx.adminctl, slot_idx )==result ); + FD_TEST( event_seq==i+1UL ); + fd_event_admin_command_t event; + fd_memcpy( &event, event_mem, sizeof(event) ); + FD_TEST( event.type==event_type && event.result==event_result ); + if( i<2UL ) { + char const * args = payload_cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? "{}" : + flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ? "{\"yes\":true,\"force\":true}" : "{\"yes\":false,\"force\":false}"; + FD_TEST( event.args_json_len==strlen( args ) && !memcmp( event.args_json, args, event.args_json_len ) ); + } + } + fd_event_tl = NULL; +} + +static void +test_failover_identity( void ) { + setup(); + FD_TEST( fd_topob_new( test_topo, "admin-test" ) ); + fd_topob_wksp( test_topo, "wksp" )->wksp = NULL; + fd_keyswitch_t * votor_ks = mock_tile( "votor" ); + fd_keyswitch_t * sign_ks = mock_tile( "sign" ); + ctx.topo = test_topo; + ctx.failover_enabled = 1; + fd_memset( ctx.junk_pubkey, 0x11, 32UL ); + fd_memset( ctx.failover_pubkey, 0x5A, 32UL ); + fd_memcpy( ctx.identity_pubkey, ctx.junk_pubkey, 32UL ); + uchar third[ 32 ]; fd_memset( third, 0x66, 32UL ); + bus_setup( 8UL ); + + /* A key other than junk and staked becomes the failover identity, and + nothing goes to the failover tile until every tile switched. */ + failover_operator_begin( &ctx, third ); + FD_TEST( ctx.failover_epoch==1UL && fd_memeq( ctx.failover_pubkey, third, 32UL ) && ctx.failover_pending && !bus_seq ); + FD_TEST( sign_ks->param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR && votor_ks->failover==0UL ); + fd_memcpy( ctx.identity_pubkey, third, 32UL ); + failover_operator_notify( &ctx, bus_stem ); + fd_failover_operator_t operator = bus_operator(); + FD_TEST( !ctx.failover_pending && operator.epoch==1UL ); + FD_TEST( fd_memeq( operator.identity, third, 32UL ) && fd_memeq( operator.failover_identity, third, 32UL ) ); + + /* The junk key keeps the failover identity. With three credits left + OPERATOR waits, and goes out once there is room. */ + bus_cr_avail = 3UL; + failover_operator_begin( &ctx, ctx.junk_pubkey ); + fd_memcpy( ctx.identity_pubkey, ctx.junk_pubkey, 32UL ); + failover_operator_notify( &ctx, bus_stem ); + FD_TEST( ctx.failover_pending && bus_seq==1UL && fd_memeq( ctx.failover_pubkey, third, 32UL ) ); + FD_TEST( fd_sha512_join( fd_sha512_new( ctx.sha512 ) ) ); + ctx.adminctl = fd_adminctl_join( fd_adminctl_new( adminctl_mem ) ); + FD_TEST( ctx.adminctl ); + void * payload; + ulong payload_max; + ulong slot_idx = fd_adminctl_reserve( ctx.adminctl, &payload, &payload_max ); + FD_TEST( slot_idx!=ULONG_MAX ); + fd_adminctl_failover_req_t creq = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_HANDOFF }; + fd_memcpy( payload, &creq, sizeof(creq) ); + fd_adminctl_publish( ctx.adminctl, slot_idx, FD_ADMINCTL_CMD_FAILOVER_PROMOTE, sizeof(creq) ); + ulong poll_idx; + void * data; + ulong data_sz; + FD_TEST( fd_adminctl_poll( ctx.adminctl, &poll_idx, &data, &data_sz )==FD_ADMINCTL_CMD_FAILOVER_PROMOTE ); + failover_request( &ctx, bus_stem, poll_idx, FD_ADMINCTL_CMD_FAILOVER_PROMOTE, data, data_sz ); + FD_TEST( bus_seq==1UL ); + FD_TEST( fd_adminctl_wait( ctx.adminctl, slot_idx )==FD_FAILOVER_CONTROL_RESULT_BUSY ); + bus_cr_avail = 8UL; + failover_operator_notify( &ctx, bus_stem ); + operator = bus_operator(); + FD_TEST( !ctx.failover_pending && operator.epoch==2UL ); + FD_TEST( fd_memeq( operator.identity, ctx.junk_pubkey, 32UL ) && fd_memeq( operator.failover_identity, third, 32UL ) ); + + /* A switch request from before set-identity gets STALE with what it + installed. */ + ctx.failov_in.nonce = 9UL; + fd_failover_switch_req_t sreq = { .epoch=1UL }; + fd_memcpy( sreq.identity, third, 32UL ); + fd_memcpy( ctx.failov_in.payload, &sreq, sizeof(sreq) ); + failover_switch_request( &ctx, bus_stem ); + fd_failover_switch_resp_t answer; + fd_memcpy( &answer, ((fd_failover_bus_msg_t const *)bus_out)->payload, sizeof(answer) ); + FD_TEST( answer.result==FD_FAILOVER_SWITCH_ERR_STALE && answer.operator.epoch==2UL ); + FD_TEST( fd_memeq( answer.operator.identity, ctx.junk_pubkey, 32UL ) && fd_memeq( answer.operator.failover_identity, third, 32UL ) ); + + /* Another key becomes the target, also while this machine runs the + current one, the CLI warns about that case. */ + uchar keypair[ 64 ]; fd_memset( keypair, 0x77, 32UL ); + fd_ed25519_public_from_private( keypair+32UL, keypair, ctx.sha512 ); + fd_memcpy( ctx.identity_pubkey, third, 32UL ); + ctx.failover_pending = 0; + FD_TEST( fd_topob_new( test_topo, "admin-test" ) ); + fd_topob_wksp( test_topo, "wksp" )->wksp = NULL; + (void)mock_tile( "votor" ); + FD_TEST( failover_prepare( &ctx, keypair )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx.failover_epoch==3UL && ctx.failover_pending && ctx.failover_retargeted ); + FD_TEST( fd_memeq( ctx.failover_pubkey, keypair+32UL, 32UL ) && fd_memeq( ctx.identity_pubkey, third, 32UL ) ); + + /* Same key is a no-op, a mismatched keypair and the junk key are refused. */ + uchar other[ 64 ]; fd_memset( other, 0x78, 32UL ); + fd_ed25519_public_from_private( other+32UL, other, ctx.sha512 ); + FD_TEST( failover_prepare( &ctx, other )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx.failover_epoch==4UL && fd_memeq( ctx.failover_pubkey, other+32UL, 32UL ) ); + FD_TEST( failover_prepare( &ctx, other )==FD_ADMINCTL_RESULT_SUCCESS && ctx.failover_epoch==4UL ); + uchar bad[ 64 ]; fd_memcpy( bad, other, 64UL ); bad[ 32 ]^=1; + FD_TEST( failover_prepare( &ctx, bad )==FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH ); + uchar junk[ 64 ]; fd_memset( junk, 0x79, 32UL ); + fd_ed25519_public_from_private( junk+32UL, junk, ctx.sha512 ); + fd_memcpy( ctx.junk_pubkey, junk+32UL, 32UL ); + FD_TEST( failover_prepare( &ctx, junk )==FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT ); + FD_TEST( ctx.failover_epoch==4UL && fd_memeq( ctx.failover_pubkey, other+32UL, 32UL ) ); + ctx.topo = NULL; +} + int main( int argc, char ** argv ) { @@ -201,6 +405,11 @@ main( int argc, test_add_authorized_voter(); test_remove_all_authorized_voters(); test_set_identity(); + test_failover_command( FD_ADMINCTL_CMD_FAILOVER_STATUS, FD_ADMINCTL_FAILOVER_CMD_STATUS, 0UL, FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_STATUS ); + test_failover_command( FD_ADMINCTL_CMD_FAILOVER_PROMOTE, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0UL, FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE ); + test_failover_command( FD_ADMINCTL_CMD_FAILOVER_PROMOTE, FD_ADMINCTL_FAILOVER_CMD_PROMOTE, + FD_ADMINCTL_FAILOVER_FLAG_YES|FD_ADMINCTL_FAILOVER_FLAG_FORCE, FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE ); + test_failover_identity(); FD_LOG_NOTICE(( "pass" )); fd_halt(); diff --git a/src/discof/failover/Local.mk b/src/discof/failover/Local.mk new file mode 100644 index 00000000000..d748b75a66e --- /dev/null +++ b/src/discof/failover/Local.mk @@ -0,0 +1,18 @@ +$(call add-hdrs,fd_failover_proto.h fd_failover_wire.h) +$(call add-objs,fd_failover_proto,fd_discof) +$(call add-objs,fd_failover_wire,fd_discof) + +ifdef FD_HAS_HOSTED +$(call add-hdrs,fd_failover_channel.h) +$(call add-objs,fd_failover_channel,fd_discof) +$(call add-objs,fd_failover_tls,fd_discof) +$(call add-objs,fd_failover_tile,fd_discof) +$(call make-unit-test,test_failover_proto,test_failover_proto,fd_discof fd_choreo fd_flamenco fd_tango fd_ballet fd_util) +$(call run-unit-test,test_failover_proto) +$(call make-unit-test,test_failover_wire,test_failover_wire,fd_discof fd_ballet fd_util) +$(call run-unit-test,test_failover_wire) +$(call make-unit-test,test_failover_channel,test_failover_channel,fd_discof fd_disco fd_choreo fd_flamenco fd_waltz fd_tls fd_tango fd_ballet fd_util) +$(call run-unit-test,test_failover_channel) +$(call make-unit-test,test_failover_tile,test_failover_tile,fd_discof fd_disco fd_choreo fd_flamenco fd_waltz fd_tls fd_tango fd_ballet fd_util) +$(call run-unit-test,test_failover_tile) +endif diff --git a/src/discof/failover/fd_failover_bus.h b/src/discof/failover/fd_failover_bus.h new file mode 100644 index 00000000000..9cc4782180d --- /dev/null +++ b/src/discof/failover/fd_failover_bus.h @@ -0,0 +1,83 @@ +#ifndef HEADER_fd_src_discof_failover_fd_failover_bus_h +#define HEADER_fd_src_discof_failover_fd_failover_bus_h + +/* The admin and failover tiles talk over the admin_failov and + failov_admin links. The admin tile stays the only adminctl poller. It + forwards failover commands with a nonce and parks the adminctl slot + until the response comes back or the deadline passes. The failover + tile asks for identity switches the same way. Every frame is one + fd_failover_bus_msg_t. Each side has at most one request outstanding, + even past the deadline, so neither link has more than two frames in + flight and neither tile can backpressure the other, even behind an + identity switch that never finishes. */ + +#include "../admin/fd_adminctl.h" + +#define FD_FAILOVER_BUS_SWITCH_REQ (1UL) /* failov to admin, fd_failover_switch_req_t */ +#define FD_FAILOVER_BUS_SWITCH_RESP (2UL) /* admin to failov, fd_failover_switch_resp_t */ +#define FD_FAILOVER_BUS_REQUEST (3UL) /* admin to failov, fd_adminctl_failover_req_t */ +#define FD_FAILOVER_BUS_RESPONSE (4UL) /* failov to admin, the requested command's response */ +#define FD_FAILOVER_BUS_OPERATOR_ACTION (5UL) /* admin to failov, fd_failover_operator_t, set-identity ran */ + +/* Only the public key goes over the bus, and the sign tile selects its + junk key or the failover identity. epoch is the failover epoch the + failover tile last saw, and the admin tile refuses a request from + before a set-identity. */ +struct fd_failover_switch_req { + uchar identity[ 32 ]; /* public key of the identity to install */ + ulong epoch; +}; + +typedef struct fd_failover_switch_req fd_failover_switch_req_t; + +#define FD_FAILOVER_SWITCH_OK (0UL) +#define FD_FAILOVER_SWITCH_ERR_DISABLED (1UL) /* failover is off here */ +#define FD_FAILOVER_SWITCH_ERR_STALE (2UL) /* set-identity ran since the request, nothing switched */ +#define FD_FAILOVER_SWITCH_ERR_AUTHORIZED_VOTER (3UL) /* the target is an authorized voter, nothing switched */ + +/* set-identity or failover promote ran, and each bumps the + failover epoch. identity is the installed public key, and + failover_identity is the key failover hands off, which set-identity + changes to identity unless that is the junk key. */ +struct fd_failover_operator { + ulong epoch; + uchar identity[ 32 ]; + uchar failover_identity[ 32 ]; +}; + +typedef struct fd_failover_operator fd_failover_operator_t; + +/* Sent once every tile has switched, or right away when refused. */ +struct fd_failover_switch_resp { + ulong result; /* FD_FAILOVER_SWITCH_* */ + ulong history_watermark; /* the votor keyswitch result, its history output sequence at the halt */ + fd_failover_operator_t operator; /* with STALE */ +}; + +typedef struct fd_failover_switch_resp fd_failover_switch_resp_t; + +#define FD_FAILOVER_BUS_PAYLOAD_MAX (256UL) + +FD_STATIC_ASSERT( sizeof(fd_failover_switch_req_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, switch_req_fits ); +FD_STATIC_ASSERT( sizeof(fd_failover_switch_resp_t)<=FD_FAILOVER_BUS_PAYLOAD_MAX, switch_resp_fits ); +FD_STATIC_ASSERT( sizeof(fd_failover_operator_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, operator_fits ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_req_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, command_fits ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)<=FD_FAILOVER_BUS_PAYLOAD_MAX, control_resp_fits ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_status_resp_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, status_resp_fits ); + +struct fd_failover_bus_msg { + ulong nonce; /* per request, echoed on the response */ + ulong result; /* the adminctl result for command responses */ + uchar payload[ FD_FAILOVER_BUS_PAYLOAD_MAX ]; +}; + +typedef struct fd_failover_bus_msg fd_failover_bus_msg_t; + +#define FD_FAILOVER_BUS_MTU (sizeof(fd_failover_bus_msg_t)) +FD_STATIC_ASSERT( sizeof(fd_failover_bus_msg_t)==272UL, bus_layout ); + +/* How long the admin tile waits for a failover command response + before it responds to the operator itself. */ +#define FD_FAILOVER_BUS_DEADLINE_NANOS (2000000000L) + +#endif /* HEADER_fd_src_discof_failover_fd_failover_bus_h */ diff --git a/src/discof/failover/fd_failover_channel.c b/src/discof/failover/fd_failover_channel.c new file mode 100644 index 00000000000..99c81ca278e --- /dev/null +++ b/src/discof/failover/fd_failover_channel.c @@ -0,0 +1,1081 @@ +#define _GNU_SOURCE +#include "fd_failover_channel.h" +#include "fd_failover_tls.h" +#include "../../ballet/base58/fd_base58.h" +#include "../../util/fd_util.h" +#include "../../util/net/fd_ip4.h" +#include "../../ballet/ed25519/fd_ed25519.h" +#include "../../waltz/tls/fd_tls_proto.h" +#include "../../waltz/fd_token_bucket.h" +#include "../../disco/bundle/fd_keepalive.h" + +#include +#include +#include +#include +#include +#include + +/* Candidate slots. */ +#define OUTBOUND_IDX (0UL) +#define ADMITTED_IDX (1UL) +#define UNAUTHENTICATED_IDX (2UL) + +/* TLS admission phase FSM states. */ +#define PHASE_CONNECT (0) +#define PHASE_START_SEND (1) +#define PHASE_START_RECV (2) +#define PHASE_CHALLENGE_SEND (3) +#define PHASE_CHALLENGE_RECV (4) +#define PHASE_REQUEST_SIGN (5) +#define PHASE_REQUEST_SEND (6) +#define PHASE_REQUEST_RECV (7) +#define PHASE_RESPONSE_SIGN (8) +#define PHASE_RESPONSE_SEND (9) +#define PHASE_RESPONSE_RECV (10) +#define PHASE_TLS_START (11) +#define PHASE_TLS (12) +#define PHASE_HELLO (13) +#define PHASE_READY (14) + +/* Bounds. */ +FD_STATIC_ASSERT( FD_FAILOVER_CHANNEL_CANDIDATE_MAX>UNAUTHENTICATED_IDX, admission_pool_capacity ); +FD_STATIC_ASSERT( FD_FAILOVER_CHANNEL_KEEPALIVE_NANOS>=2L && FD_FAILOVER_CHANNEL_IDLE_NANOS>0L, keepalive_params ); + +FD_STATIC_ASSERT( FD_KEYGUARD_FAILOVER_PROOF_SZ<=FD_TLS_CV_SIGN_SZ, admission_sign_mtu ); +FD_STATIC_ASSERT( FD_FAILOVER_CLIENT_HELLO_MAX+sizeof(fd_tlsrec_hdr_t)+sizeof(fd_failover_admission_start_t)<=FD_FAILOVER_ADMISSION_MAX, admission_buffer ); + +/* A HELLO whose junk key does not match the key that was TLS authenticated */ +#define HELLO_ERR_TLS_KEY (-1) + +struct candidate { + int fd; /* -1 for an unused candidate. */ + int phase; /* PHASE_* */ + int dialed; /* we dialed it, else the listener accepted it */ + uint address; + long deadline; + long admission_until; + int tls_started; /* tls is initialized, close_candidate must fini it */ + fd_failover_hello_t local_hello; + fd_failover_admission_start_t start; + fd_failover_admission_challenge_t challenge; + uchar client_hello[ FD_FAILOVER_CLIENT_HELLO_MAX+sizeof(fd_tlsrec_hdr_t) ]; + uchar transcript[ 32 ]; + uchar client_proof[ 64 ]; + fd_failover_tls_t tls; + fd_failover_hello_t hello; + uchar rx[ FD_FAILOVER_FRAME_MAX ]; + uchar tx[ FD_FAILOVER_FRAME_MAX ]; + ulong rx_used; + ulong tx_used; + ulong tx_sent; +}; + +struct fd_failover_channel { + /* Session */ + ulong magic; /* FD_FAILOVER_CHANNEL_MAGIC once initialized */ + int dial_peer; /* the session machine runs as the dialer */ + uint peer_addr; /* the dial destination, zero when we do not dial */ + ushort peer_port; /* the dial destination port */ + int listen_fd; /* listening socket, -1 until it is open */ + int paired_idx; /* the paired candidate, -1 while none is paired */ + ulong cursor; /* the next unauthenticated candidate idx to service */ + + /* Credentials */ + fd_failover_tls_ctx_t tls_ctx; /* TLS template every connection starts from */ + fd_failover_hello_t self_hello; /* the HELLO we send */ + fd_failover_hello_t peer_hello; /* the paired peer's HELLO, zero while unpaired */ + fd_failover_proof_signer_t proof_signer; /* signs our admission proofs in the sign tile */ + fd_sha512_t sha[ 1 ]; /* scratch hash for checking admission proofs */ + + /* Connections */ + struct candidate candidates[ FD_FAILOVER_CHANNEL_CANDIDATE_MAX ]; /* connections that are handshaking or paired */ + uint known_addr; /* the last address that authenticated, gets the ADMITTED slot ahead of admission */ + long known_until; /* until when */ + + /* Rate limits */ + fd_token_bucket_t accept_bucket; /* accept syscalls */ + fd_token_bucket_t admit_bucket; /* candidates started */ + fd_token_bucket_t verify_bucket; /* Ed25519 proof checks and signs, at most one per poll */ + int verify_budget; /* proof checks left this poll */ + + /* Timers */ + fd_keepalive_t keepalive[ 1 ]; /* when the paired session last heard from the peer / pings it next */ + fd_rng_t rng[ 1 ]; /* spreads redials and KEEPALIVEs, not for secure RNG */ + long retry_at; /* when the dialer dials again */ + long paired_at; /* when the current session paired */ + long backoff; /* the current dial backoff, doubles up to BACKOFF_MAX */ + + /* The paired session */ + int expected_close; /* RESULT was sent, peer may close normally */ + ulong generation; /* identifies the current authenticated session */ +}; + +FD_FN_CONST ulong fd_failover_channel_align ( void ) { return alignof(fd_failover_channel_t); } +FD_FN_CONST ulong fd_failover_channel_footprint( void ) { return sizeof (fd_failover_channel_t); } + +/* A bucket of burst tokens refilling at `rate` per second. */ +static fd_token_bucket_t +token_bucket_new( ulong rate, + ulong burst ) { + return (fd_token_bucket_t){ .rate=(float)rate/(float)FD_FAILOVER_NANOS_PER_SECOND, .burst=(float)burst, .balance=(float)burst }; +} + +void * +fd_failover_channel_new( void * shmem ) { + if( !shmem || !fd_ulong_is_aligned( (ulong)shmem, fd_failover_channel_align() ) ) return NULL; + fd_failover_channel_t * ch = shmem; + + /* Initialize the listening state. */ + fd_memset( ch, 0, sizeof(*ch) ); + ch->listen_fd = -1; + ch->paired_idx = -1; + for( ulong i=0; icandidates[i].fd = -1; + ch->backoff = FD_FAILOVER_CHANNEL_BACKOFF_MIN_NANOS; + + /* Rate limiting */ + ch->accept_bucket = token_bucket_new( FD_FAILOVER_ACCEPT_RATE, FD_FAILOVER_ACCEPT_BURST ); + ch->admit_bucket = token_bucket_new( FD_FAILOVER_ADMIT_RATE, FD_FAILOVER_ADMIT_BURST ); + ch->verify_bucket = token_bucket_new( FD_FAILOVER_VERIFY_RATE, FD_FAILOVER_VERIFY_BURST ); + + /* Set up secure randomness */ + uint seed; + FD_TEST( fd_rng_secure( &seed, sizeof(seed) ) ); + FD_TEST( fd_rng_join( fd_rng_new( ch->rng, seed, 0UL ) ) ); + FD_TEST( fd_sha512_join( fd_sha512_new( ch->sha ) ) ); + + ch->magic = FD_FAILOVER_CHANNEL_MAGIC; + return shmem; +} + +fd_failover_channel_t * +fd_failover_channel_join( void * shmem ) { + if( !shmem || !fd_ulong_is_aligned( (ulong)shmem, fd_failover_channel_align() ) ) return NULL; + fd_failover_channel_t * ch = shmem; + return ch->magic==FD_FAILOVER_CHANNEL_MAGIC ? ch : NULL; +} + +static void +close_candidate( struct candidate * c ) { + if( c->tls_started ) fd_failover_tls_fini( &c->tls ); + c->tls_started = 0; + if( c->fd!=-1 ) close( c->fd ); + /* Closing a candidate frees its slot. */ + c->fd = -1; + c->dialed = 0; + c->rx_used = c->tx_used = c->tx_sent = 0UL; +} + +/* Discards every candidate and the pairing. */ +static void +discard( fd_failover_channel_t * ch ) { + for( ulong i=0; icandidates[i] ); + ch->paired_idx = -1; + fd_memset( &ch->peer_hello, 0, sizeof(ch->peer_hello) ); +} + +static int +want_dial( fd_failover_channel_t const * ch ) { + return !!ch->peer_addr; +} + +/* Resets an unpaired channel: drops any dial in flight and goes back + to listening, or to an immediate redial when a dial address is set. */ +static void +reset( fd_failover_channel_t * ch ) { + if( ch->paired_idx>=0 ) return; + for( ulong i=0; icandidates[i].dialed ) { + close_candidate( &ch->candidates[i] ); + } + } + ch->dial_peer = want_dial( ch ); + ch->retry_at = 0L; +} + +void +fd_failover_channel_init_listener( fd_failover_channel_t * ch, + uint address, + ushort port ) { + discard( ch ); + reset( ch ); + if( ch->listen_fd!=-1 ) return; + ch->listen_fd = socket( AF_INET, SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC, 0 ); + if( ch->listen_fd==-1 ) { + FD_LOG_ERR(( "failover socket failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } + int yes = 1; + if( setsockopt( ch->listen_fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes) ) ) { + FD_LOG_ERR(( "failover setsockopt failed" )); + } + if( setsockopt( ch->listen_fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes) ) ) { + FD_LOG_ERR(( "failover setsockopt(TCP_NODELAY) failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } + struct sockaddr_in addr = { .sin_family=AF_INET, .sin_port=fd_ushort_bswap( port ), .sin_addr.s_addr=address }; + if( bind( ch->listen_fd, fd_type_pun( &addr ), sizeof(addr) ) || listen( ch->listen_fd, FD_FAILOVER_LISTEN_BACKLOG ) ) { + FD_LOG_ERR(( "could not listen on `" FD_IP4_ADDR_FMT ":%hu` (%i-%s)", + FD_IP4_ADDR_FMT_ARGS( address ), port, errno, fd_io_strerror( errno ) )); + } +} + +void +fd_failover_channel_init_dialer( fd_failover_channel_t * ch, + uint address, + ushort port ) { + if( address==ch->peer_addr && port==ch->peer_port ) return; + if( address && ch->paired_idx>=0 ) discard( ch ); + ch->peer_addr = address; + ch->peer_port = port; + if( address ) { + ch->backoff = FD_FAILOVER_CHANNEL_BACKOFF_MIN_NANOS; + } + reset( ch ); +} + +int +fd_failover_channel_set_identity( fd_failover_channel_t * ch, + uchar const junk_pubkey[ 32 ], + fd_tls_sign_t signer, + fd_failover_proof_signer_t proof_signer, + fd_failover_hello_t const * hello ) { + if( !proof_signer.sign_fn || !signer.sign_fn || + !fd_memeq( junk_pubkey, hello->junk_pubkey, 32UL ) || + fd_memeq( hello->junk_pubkey, hello->staked_pubkey, 32UL ) ) return -1; + discard( ch ); + /* An identity change requires a fresh session. */ + ch->known_addr = 0U; + ch->known_until = 0L; + ch->self_hello = *hello; + ch->proof_signer = proof_signer; + return fd_failover_tls_ctx_init( &ch->tls_ctx, junk_pubkey, signer ); +} + +void +fd_failover_channel_fini( fd_failover_channel_t * ch ) { + discard( ch ); + if( ch->listen_fd!=-1 ) close( ch->listen_fd ); + ch->listen_fd = -1; + fd_failover_tls_ctx_fini( &ch->tls_ctx ); +} + +void +fd_failover_channel_set_holds( fd_failover_channel_t * ch, + int holds ) { + if( ch->self_hello.holds==(uchar)!!holds ) return; + ch->self_hello.holds = (uchar)!!holds; + for( ulong i=0UL; ipaired_idx ) close_candidate( &ch->candidates[i] ); + if( ch->paired_idx<0 ) reset( ch ); +} + +/* Derive the session state from the socket state. */ +FD_FN_PURE ulong +fd_failover_channel_state( fd_failover_channel_t const * ch ) { + if( ch->paired_idx>=0 ) return FD_FAILOVER_SESSION_PAIRED; + if( !ch->dial_peer ) return FD_FAILOVER_SESSION_LISTENING; + struct candidate const * c = &ch->candidates[OUTBOUND_IDX]; + if( c->fd==-1 ) return FD_FAILOVER_SESSION_BACKOFF; + if( c->phase==PHASE_CONNECT ) return FD_FAILOVER_SESSION_DIALING; + return FD_FAILOVER_SESSION_HELLO; +} + +FD_FN_PURE int fd_failover_channel_listen_fd( fd_failover_channel_t const * ch ) { return ch->listen_fd; } + +FD_FN_PURE int +fd_failover_channel_tx_pending( fd_failover_channel_t const * ch ) { + if( ch->paired_idx<0 ) return 0; + struct candidate const * c = &ch->candidates[ch->paired_idx]; + return !!c->tx_used || fd_tlsrec_sock_tx_pending( &c->tls.sock ); +} + +FD_FN_PURE fd_failover_hello_t const * fd_failover_channel_peer_hello( fd_failover_channel_t const * ch ) { return &ch->peer_hello; } +FD_FN_PURE ulong fd_failover_channel_generation( fd_failover_channel_t const * ch ) { return ch->generation; } + +ushort +fd_failover_channel_listen_port( fd_failover_channel_t const * ch ) { + struct sockaddr_in addr; + socklen_t len = sizeof(addr); + if( getsockname( ch->listen_fd, fd_type_pun( &addr ), &len ) ) FD_LOG_ERR(( "failover getsockname failed" )); + return fd_ushort_bswap( addr.sin_port ); +} + +static void +drop( fd_failover_channel_t * ch, + ulong idx, + long now ) { + struct candidate * c = &ch->candidates[idx]; + int paired = ch->paired_idx==(int)idx; + int accepted = c->fd!=-1 && !c->dialed; + close_candidate( c ); + if( paired ) { + ch->paired_idx = -1; + fd_memset( &ch->peer_hello, 0, sizeof(ch->peer_hello) ); + } + /* A candidate from the listener does not get a session until it pairs. */ + if( !paired && accepted ) return; + /* After loss, use the current handoff address to choose listener or dialer. */ + if( paired && want_dial( ch )!=ch->dial_peer ) reset( ch ); + if( ch->dial_peer ) { + /* Recompute backoff parameters */ + int established = paired && now>=ch->paired_at && now-ch->paired_at>=FD_FAILOVER_CHANNEL_HELLO_TIMEOUT_NANOS; + long jitter = (long)fd_rng_ulong_roll( ch->rng, (ulong)FD_FAILOVER_CHANNEL_BACKOFF_MIN_NANOS ); + ch->retry_at = fd_long_sat_add( now, fd_long_sat_add( established ? 0L : ch->backoff, jitter ) ); + ch->backoff = established ? FD_FAILOVER_CHANNEL_BACKOFF_MIN_NANOS + : fd_long_min( fd_long_sat_add( ch->backoff, ch->backoff ), FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS ); + } +} + +/* Same as drop, and if idx is the paired session we log why it ended. */ +static void +lose( fd_failover_channel_t * ch, + ulong idx, + long now, + char const * why ) { + if( ch->paired_idx==(int)idx ) { + FD_LOG_WARNING(( "lost the failover session with `" FD_IP4_ADDR_FMT "`, %s", + FD_IP4_ADDR_FMT_ARGS( ch->candidates[idx].address ), why )); + } + drop( ch, idx, now ); +} + +void +fd_failover_channel_hangup( fd_failover_channel_t * ch, + long now ) { + if( ch->paired_idx>=0 ) drop( ch, (ulong)ch->paired_idx, now ); + for( ulong i=0; icandidates[i].fd!=-1 ) drop( ch, i, now ); +} + +void +fd_failover_channel_protocol_error( fd_failover_channel_t * ch, + long now ) { + if( FD_LIKELY( ch->paired_idx>=0 ) ) { + lose( ch, (ulong)ch->paired_idx, now, "it sent a message that breaks the protocol" ); + } + fd_failover_channel_hangup( ch, now ); +} + +/* Admit at most one live inbound candidate per source address. */ +static int +admit( fd_failover_channel_t const * ch, + uint address ) { + for( ulong i=ADMITTED_IDX; icandidates[i].fd!=-1 && ch->candidates[i].address==address ) return 0; + return 1; +} + +static int +has_proof( struct candidate const * c ) { + return c->phase==PHASE_REQUEST_RECV && c->rx_used==sizeof(c->client_proof); +} + +static int +known_source( fd_failover_channel_t const * ch, uint address, long now ) { + return address==ch->known_addr && nowknown_until; +} + +static void +remember_source( fd_failover_channel_t * ch, uint address, long now ) { + ch->known_addr = address; + ch->known_until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); +} + +static ulong +listen_slot( fd_failover_channel_t const * ch, + uint address, + long now ) { + if( known_source( ch, address, now ) ) { + return ch->candidates[ADMITTED_IDX].fd==-1 ? ADMITTED_IDX : FD_FAILOVER_CHANNEL_CANDIDATE_MAX; + } + for( ulong i=UNAUTHENTICATED_IDX; icandidates[i].fd==-1 ) return i; + } + return FD_FAILOVER_CHANNEL_CANDIDATE_MAX; +} + +static void +start( fd_failover_channel_t * ch, + ulong idx, + int fd, + uint address, + long now, + int phase ) { + struct candidate * c = &ch->candidates[idx]; + /* Inbound sockets do not initialize TLS until admission succeeds. */ + c->fd = fd; + c->address = address; + c->phase = phase; + c->dialed = phase==PHASE_CONNECT; + c->deadline = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_HELLO_TIMEOUT_NANOS ); + c->admission_until = c->dialed ? c->deadline : fd_long_min( c->deadline, fd_long_sat_add( now, FD_FAILOVER_ADMISSION_START_NANOS ) ); + c->local_hello = ch->self_hello; + c->rx_used = c->tx_used = c->tx_sent = 0UL; + c->tls_started = 0; +} + +static void +accept_candidates( fd_failover_channel_t * ch, + long now, + int * busy ) { + for( ulong n=0; naccept_bucket, 1.0f, now ) ) return; + struct sockaddr_in addr; + socklen_t len = sizeof(addr); + int fd = accept4( ch->listen_fd, fd_type_pun( &addr ), &len, SOCK_NONBLOCK|SOCK_CLOEXEC ); + if( fd==-1 ) return; + *busy = 1; + uint address = addr.sin_addr.s_addr; + ulong idx = listen_slot( ch, address, now ); + if( ch->paired_idx>=0 || len!=sizeof(addr) || addr.sin_family!=AF_INET || + idx==FD_FAILOVER_CHANNEL_CANDIDATE_MAX || !admit( ch, address ) || + !fd_token_bucket_consume( &ch->admit_bucket, 1.0f, now ) ) { + close( fd ); + continue; + } + start( ch, idx, fd, address, now, PHASE_START_RECV ); + } +} + +static int +flush( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy ) { + struct candidate * c = &ch->candidates[idx]; + if( !c->tx_used ) { + int rc = fd_tlsrec_sock_flush( &c->tls.sock, c->tls.fd ); + if( rc<0 ) { lose( ch, idx, now, "flushing TLS output failed" ); return -1; } + return rc; + } + long n = fd_failover_tls_write( &c->tls, c->tx+c->tx_sent, c->tx_used-c->tx_sent ); + if( n<0L ) { lose( ch, idx, now, "writing to it failed" ); return -1; } + if( !n ) return 1; + *busy = 1; + c->tx_sent += (ulong)n; + if( c->tx_senttx_used ) return 1; + c->tx_used = c->tx_sent = 0UL; + return fd_tlsrec_sock_tx_pending( &c->tls.sock ) ? 1 : 0; +} + +static int +decode( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy, + ushort * type, + uchar * payload, + ulong * payload_sz ) { + struct candidate * c = &ch->candidates[idx]; + uchar const * data; + ulong frame_sz; + int err = fd_failover_wire_decode( c->rx, c->rx_used, type, &data, payload_sz, &frame_sz ); + if( err==FD_FAILOVER_WIRE_ERR_AGAIN ) return 0; + if( err ) { lose( ch, idx, now, "it sent a malformed frame" ); return -1; } + fd_memcpy( payload, data, *payload_sz ); + memmove( c->rx, c->rx+frame_sz, c->rx_used-frame_sz ); + c->rx_used -= frame_sz; + *busy = 1; + return 1; +} + +static char const * +hello_err_cstr( int err ) { + switch( err ) { + case HELLO_ERR_TLS_KEY: return "its junk key is not the key TLS authenticated"; + case FD_FAILOVER_HELLO_ERR_VERSION: return "the protocol version differs"; + case FD_FAILOVER_HELLO_ERR_STAKED: return "the failover identity differs"; + case FD_FAILOVER_HELLO_ERR_VOTE_ACCT: return "the vote account differs"; + case FD_FAILOVER_HELLO_ERR_JUNK_EQ: return "its junk key is ours"; + case FD_FAILOVER_HELLO_ERR_JUNK_STAKE: return "a junk key is the identity being handed off"; + case FD_FAILOVER_HELLO_ERR_BOTH_HOLD: return "both machines run the identity"; + case FD_FAILOVER_HELLO_ERR_HOLDS: return "it sent an unknown value"; + case FD_FAILOVER_HELLO_ERR_BOOT_ID: return "the boot id is zero"; + case FD_FAILOVER_HELLO_ERR_MODE: return "the consensus mode differs"; + case FD_FAILOVER_HELLO_ERR_PROOF: return "its admission proof is not signed by the failover identity"; + default: return "unknown"; + } +} + +static void +refuse_hello( uint address, int err ) { + FD_LOG_WARNING(( "rejected the failover peer at `" FD_IP4_ADDR_FMT "`, %s", + FD_IP4_ADDR_FMT_ARGS( address ), hello_err_cstr( err ) )); +} + +static int +read_frame( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy, + ushort * type, + uchar * payload, + ulong * payload_sz ) { + struct candidate * c = &ch->candidates[idx]; + int rc = decode( ch, idx, now, busy, type, payload, payload_sz ); + if( rc ) return rc; + ulong max = ch->paired_idx==(int)idx ? sizeof(c->rx) : FD_FAILOVER_FRAME_HDR_SZ+sizeof(fd_failover_hello_t); + if( c->rx_used>=max ) { drop( ch, idx, now ); return -1; } + long n = fd_failover_tls_read( &c->tls, c->rx+c->rx_used, max-c->rx_used ); + if( n<0L ) { + if( FD_UNLIKELY( c->dialed && c->phase==PHASE_HELLO ) ) { + FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` closed the connection before answering our HELLO", + FD_IP4_ADDR_FMT_ARGS( c->address ) )); + } + if( ch->paired_idx==(int)idx && ch->expected_close && c->tls.peer_closed && !fd_failover_channel_tx_pending( ch ) ) { + drop( ch, idx, now ); + return -1; + } + if( fd_tlsrec_conn_is_failed( &c->tls.conn ) ) { + char why[ 192 ]; + fd_cstr_printf( why, sizeof(why), NULL, "TLS record rejected (%s)", + fd_tls_reason_cstr( c->tls.conn.hs.base.reason ) ); + lose( ch, idx, now, why ); + } else { + lose( ch, idx, now, "the connection closed" ); + } + return -1; + } + if( n>0L ) { c->rx_used += (ulong)n; *busy = 1; } + return decode( ch, idx, now, busy, type, payload, payload_sz ); +} + +/* Return one to continue this poll, zero to stop after a wait or failure. */ +static int +connect_candidate( fd_failover_channel_t * ch, + ulong idx, + long now ) { + struct candidate * c = &ch->candidates[idx]; + struct sockaddr_in addr = { + .sin_family=AF_INET, + .sin_port=fd_ushort_bswap( ch->peer_port ), + .sin_addr.s_addr=ch->peer_addr + }; + if( connect( c->fd, fd_type_pun( &addr ), sizeof(addr) ) && errno!=EISCONN ) { + if( errno==EINPROGRESS || errno==EALREADY || errno==EINTR ) return 0; /* TCP is still connecting. */ + FD_LOG_WARNING(( "could not connect to the failover peer at `" FD_IP4_ADDR_FMT ":%hu` (%i-%s)", + FD_IP4_ADDR_FMT_ARGS( ch->peer_addr ), ch->peer_port, errno, fd_io_strerror( errno ) )); + drop( ch, idx, now ); + return 0; + } + + /* Only operator-initiated outbound work prepares a TLS key share here. */ + ulong hello_sz = sizeof(c->client_hello); + if( fd_failover_tls_new( &c->tls, &ch->tls_ctx, c->fd, 1 ) ) { + drop( ch, idx, now ); + return 0; + } + c->tls_started = 1; + + if( fd_failover_tls_client_hello( &c->tls, c->client_hello, &hello_sz ) || + hello_sz>sizeof(c->client_hello) ) { + drop( ch, idx, now ); + return 0; + } + + c->start = (fd_failover_admission_start_t){ + .magic=FD_FAILOVER_ADMISSION_MAGIC, + .hello=c->local_hello, + .client_hello_sz=(ushort)hello_sz + }; + if( !fd_rng_secure( c->start.nonce, sizeof(c->start.nonce) ) ) { + drop( ch, idx, now ); + return 0; + } + fd_memcpy( c->tx, &c->start, sizeof(c->start) ); + fd_memcpy( c->tx+sizeof(c->start), c->client_hello, hello_sz ); + c->tx_used = sizeof(c->start)+hello_sz; + c->phase = PHASE_START_SEND; + return 1; +} + +static int +handshake_candidate( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy ) { + struct candidate * c = &ch->candidates[idx]; + int rc = fd_failover_tls_handshake( &c->tls ); + if( rc<0 ) { + uint reason = c->tls.conn.hs.base.reason; + FD_LOG_WARNING(( "TLS handshake with `" FD_IP4_ADDR_FMT "` failed (%s)", + FD_IP4_ADDR_FMT_ARGS( c->address ), reason ? fd_tls_reason_cstr( reason ) : "connection closed or socket error" )); + drop( ch, idx, now ); + return 0; + } + if( !rc ) return 0; /* Keep waiting for TLS. */ + *busy = 1; + + /* TLS authenticated a key, send the HELLO! */ + c->phase = PHASE_HELLO; + if( c->dialed ) { + c->tx_used = fd_failover_wire_encode( c->tx, FD_FAILOVER_MSG_HELLO, + (uchar const *)&c->local_hello, sizeof(c->local_hello) ); + } + return 1; +} + +static int +read_candidate_hello( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy, + uchar * payload ) { + struct candidate * c = &ch->candidates[idx]; + ushort type; + ulong sz; + if( read_frame( ch, idx, now, busy, &type, payload, &sz )<=0 ) return 0; + if( type!=FD_FAILOVER_MSG_HELLO || sz!=sizeof(c->hello) ) { + FD_LOG_WARNING(( "rejected failover HELLO from `" FD_IP4_ADDR_FMT "`", FD_IP4_ADDR_FMT_ARGS( c->address ) )); + drop( ch, idx, now ); + return 0; + } + + /* HELLO and TLS must match admission. */ + int err = fd_memeq( payload, &c->hello, sizeof(c->hello) ) && + fd_memeq( c->hello.junk_pubkey, c->tls.peer_pubkey, 32UL ) + ? fd_failover_hello_check( &c->local_hello, &c->hello ) : HELLO_ERR_TLS_KEY; + if( err!=FD_FAILOVER_HELLO_OK ) { + refuse_hello( c->address, err ); + drop( ch, idx, now ); + return 0; + } + + /* The peer is authenticated. */ + c->phase = PHASE_READY; + if( !c->dialed ) { + c->tx_used = fd_failover_wire_encode( c->tx, FD_FAILOVER_MSG_HELLO, + (uchar const *)&c->local_hello, sizeof(c->local_hello) ); + if( flush( ch, idx, now, busy )!=0 ) return 0; /* Remain READY until output drains, unless the write fails. */ + } + return 1; +} + +static void +pair_candidate( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy ) { + struct candidate * c = &ch->candidates[idx]; + /* The first session that authenticates wins. */ + for( ulong i=0; icandidates[i] ); + + if( !c->dialed ) ch->dial_peer = 0; + + ch->paired_idx = (int)idx; + c->tls.paired = 1; + ch->peer_hello = c->hello; + ch->generation++; + remember_source( ch, c->address, now ); + + fd_keepalive_init( ch->keepalive, ch->rng, FD_FAILOVER_CHANNEL_KEEPALIVE_NANOS, FD_FAILOVER_CHANNEL_IDLE_NANOS, now ); + fd_keepalive_rx( ch->keepalive, now ); + ch->paired_at = now; + ch->expected_close = 0; + *busy = 1; + + char member[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( c->hello.junk_pubkey, NULL, member ); + FD_LOG_NOTICE(( "paired with the failover peer at `" FD_IP4_ADDR_FMT "` via %s, authenticated member %s, %s, boot id %016lx", + FD_IP4_ADDR_FMT_ARGS( c->address ), c->dialed ? "outbound dial" : "listener", member, + c->hello.holds ? "it runs the identity" : "it does not run the identity", c->hello.boot_id )); +} + +static int +read_admission( struct candidate * c, ulong goal, int * busy ) { + if( FD_UNLIKELY( goal>FD_FAILOVER_ADMISSION_MAX || c->rx_used>goal ) ) return -1; + if( c->rx_used==goal ) return 1; + long n = recv( c->fd, c->rx+c->rx_used, goal-c->rx_used, MSG_DONTWAIT|MSG_NOSIGNAL ); + if( n<0L ) return ( errno==EAGAIN || errno==EWOULDBLOCK || errno==EINTR ) ? 0 : -1; + if( !n ) return -1; + + c->rx_used += (ulong)n; + *busy = 1; + return c->rx_used==goal; +} + +static int +write_admission( struct candidate * c, int * busy ) { + long n = send( c->fd, c->tx+c->tx_sent, c->tx_used-c->tx_sent, MSG_DONTWAIT|MSG_NOSIGNAL ); + if( n<0L ) return ( errno==EAGAIN || errno==EWOULDBLOCK || errno==EINTR ) ? 0 : -1; + if( !n ) return -1; + c->tx_sent += (ulong)n; + *busy = 1; + if( c->tx_senttx_used ) return 0; + c->tx_sent = c->tx_used = 0UL; + return 1; +} + +static int +verify_admission_proof( fd_failover_channel_t * ch, struct candidate * c, int server, uchar const sig[ 64 ] ) { + uchar msg[ FD_KEYGUARD_FAILOVER_PROOF_SZ ]; + fd_failover_proof_msg( msg, server, c->hello.junk_pubkey, c->transcript ); + return fd_ed25519_verify( msg, sizeof(msg), sig, c->local_hello.staked_pubkey, ch->sha )==FD_ED25519_SUCCESS; +} + +static int +sign_admission_proof( fd_failover_channel_t * ch, struct candidate * c, int server ) { + uchar msg[ FD_KEYGUARD_FAILOVER_PROOF_SZ ]; + fd_failover_proof_msg( msg, server, c->local_hello.junk_pubkey, c->transcript ); + ch->proof_signer.sign_fn( ch->proof_signer.ctx, c->tx, msg ); + if( fd_ed25519_verify( msg, sizeof(msg), c->tx, c->local_hello.staked_pubkey, ch->sha )!=FD_ED25519_SUCCESS ) return 0; + c->tx_used = 64UL; + c->tx_sent = 0UL; + return 1; +} + +/* One Ed25519 operation per poll, within the global rate. */ +static int +verify_credit( fd_failover_channel_t * ch, long now ) { + if( !ch->verify_budget || !fd_token_bucket_consume( &ch->verify_bucket, 1.0f, now ) ) return 0; + ch->verify_budget--; + return 1; +} + +/* Service the candidate admission state machine. */ +static void +service_admission( fd_failover_channel_t * ch, ulong idx, long now, int * busy ) { + struct candidate * c = &ch->candidates[idx]; + int rc; + switch( c->phase ) { + case PHASE_START_SEND: + case PHASE_CHALLENGE_SEND: + case PHASE_REQUEST_SEND: + case PHASE_RESPONSE_SEND: + rc = write_admission( c, busy ); + if( rc<0 ) break; + if( !rc ) return; + c->rx_used = 0UL; + if( c->phase==PHASE_START_SEND ) { + c->phase = PHASE_CHALLENGE_RECV; + } + else if( c->phase==PHASE_CHALLENGE_SEND ) { + c->admission_until = fd_long_min( c->deadline, fd_long_sat_add( now, FD_FAILOVER_ADMISSION_PROOF_NANOS ) ); + c->phase = PHASE_REQUEST_RECV; + } + else if( c->phase==PHASE_REQUEST_SEND ) { + c->phase = PHASE_RESPONSE_RECV; + } + else { + c->phase = PHASE_TLS; + } + return; + case PHASE_START_RECV: { + /* Read only the fixed header until its length is checked. */ + ulong goal = c->rx_usedstart) ? sizeof(c->start) : sizeof(c->start)+c->start.client_hello_sz; + rc = read_admission( c, goal, busy ); + if( rc<0 ) break; + if( !rc ) return; + if( goal==sizeof(c->start) ) { + fd_memcpy( &c->start, c->rx, sizeof(c->start) ); + if( c->start.magic!=FD_FAILOVER_ADMISSION_MAGIC || + c->start.client_hello_sz<=sizeof(fd_tlsrec_hdr_t) || + c->start.client_hello_sz>sizeof(c->client_hello) ) break; + int err = fd_failover_hello_check( &c->local_hello, &c->start.hello ); + if( err ) { refuse_hello( c->address, err ); break; } + return; + } + fd_memcpy( c->client_hello, c->rx+sizeof(c->start), c->start.client_hello_sz ); + fd_tlsrec_hdr_t hdr; + fd_memcpy( &hdr, c->client_hello, sizeof(hdr) ); + if( hdr.content_type!=FD_TLS_REC_HANDSHAKE || + (ulong)fd_ushort_bswap( hdr.length )+sizeof(hdr)!=c->start.client_hello_sz || + c->client_hello[ sizeof(hdr) ]!=FD_TLS_MSG_CLIENT_HELLO ) break; + c->hello = c->start.hello; + c->challenge = (fd_failover_admission_challenge_t){ .magic=FD_FAILOVER_ADMISSION_MAGIC, .hello=c->local_hello }; + if( !fd_rng_secure( c->challenge.nonce, sizeof(c->challenge.nonce) ) ) break; + fd_failover_admission_hash( &c->start, c->client_hello, &c->challenge, c->transcript ); + fd_memcpy( c->tx, &c->challenge, sizeof(c->challenge) ); + c->tx_used = sizeof(c->challenge); + c->rx_used = 0UL; + c->admission_until = fd_long_min( c->deadline, fd_long_sat_add( now, FD_FAILOVER_ADMISSION_START_NANOS ) ); + c->phase = PHASE_CHALLENGE_SEND; + return; + } + case PHASE_CHALLENGE_RECV: + rc = read_admission( c, sizeof(c->challenge), busy ); + if( rc<0 ) break; + if( !rc ) return; + fd_memcpy( &c->challenge, c->rx, sizeof(c->challenge) ); + if( c->challenge.magic!=FD_FAILOVER_ADMISSION_MAGIC ) break; + int err = fd_failover_hello_check( &c->local_hello, &c->challenge.hello ); + if( err ) { refuse_hello( c->address, err ); break; } + c->hello = c->challenge.hello; + fd_failover_admission_hash( &c->start, c->client_hello, &c->challenge, c->transcript ); + c->rx_used = 0UL; + c->phase = PHASE_REQUEST_SIGN; + *busy = 1; + return; + case PHASE_REQUEST_SIGN: + if( !verify_credit( ch, now ) ) return; + if( !sign_admission_proof( ch, c, 0 ) ) break; + fd_memcpy( c->client_proof, c->tx, sizeof(c->client_proof) ); + c->phase = PHASE_REQUEST_SEND; + *busy = 1; + return; + case PHASE_RESPONSE_RECV: { + fd_failover_admission_response_t response; + if( c->rx_usedrx, sizeof(response) ); + if( !response.server_flight_sz || response.server_flight_sz>FD_FAILOVER_SERVER_FLIGHT_MAX ) break; + return; + } + fd_memcpy( &response, c->rx, sizeof(response) ); + rc = read_admission( c, sizeof(response)+response.server_flight_sz, busy ); + if( rc<0 ) break; + if( !rc || !verify_credit( ch, now ) ) return; + fd_failover_response_hash( c->transcript, c->client_proof, c->rx+sizeof(response), + response.server_flight_sz, c->transcript ); + if( !verify_admission_proof( ch, c, 1, response.proof ) ) { + refuse_hello( c->address, FD_FAILOVER_HELLO_ERR_PROOF ); + break; + } + if( fd_failover_tls_accept_flight( &c->tls, c->rx+sizeof(response), response.server_flight_sz ) ) break; + c->phase = PHASE_TLS; + c->rx_used = 0UL; + *busy = 1; + return; + } + case PHASE_REQUEST_RECV: + rc = read_admission( c, 64UL, busy ); + if( rc<0 ) break; + if( !rc || !verify_credit( ch, now ) ) return; + if( !verify_admission_proof( ch, c, 0, c->rx ) ) { + refuse_hello( c->address, FD_FAILOVER_HELLO_ERR_PROOF ); + break; + } + fd_memcpy( c->client_proof, c->rx, sizeof(c->client_proof) ); + c->rx_used = 0UL; + c->admission_until = c->deadline; + c->phase = PHASE_TLS_START; + /* Only a verified proof can claim a protected slot. */ + if( idx!=ADMITTED_IDX ) { + if( ch->candidates[ ADMITTED_IDX ].fd!=-1 ) break; + ch->candidates[ ADMITTED_IDX ] = *c; + c->fd = -1; + c->tls_started = 0; + } + *busy = 1; + return; + case PHASE_RESPONSE_SIGN: { + if( !verify_credit( ch, now ) ) return; + fd_failover_admission_response_t response = { .server_flight_sz=(ushort)(c->tx_used-sizeof(fd_failover_admission_response_t)) }; + if( !sign_admission_proof( ch, c, 1 ) ) break; + fd_memcpy( response.proof, c->tx, sizeof(response.proof) ); + fd_memcpy( c->tx, &response, sizeof(response) ); + c->tx_used = sizeof(response)+response.server_flight_sz; + c->phase = PHASE_RESPONSE_SEND; + *busy = 1; + return; + } + case PHASE_TLS_START: + if( fd_failover_tls_new( &c->tls, &ch->tls_ctx, c->fd, 0 ) ) break; + c->tls_started = 1; + if( fd_failover_tls_accept_flight( &c->tls, c->client_hello, c->start.client_hello_sz ) ) break; + if( !c->tls.sock.tx_sz || c->tls.sock.tx_sz>FD_FAILOVER_SERVER_FLIGHT_MAX ) break; + fd_failover_response_hash( c->transcript, c->client_proof, c->tls.sock.tx_buf, + (ushort)c->tls.sock.tx_sz, c->transcript ); + fd_memcpy( c->tx+sizeof(fd_failover_admission_response_t), c->tls.sock.tx_buf, c->tls.sock.tx_sz ); + c->tx_used = sizeof(fd_failover_admission_response_t)+c->tls.sock.tx_sz; + c->tls.sock.tx_off = c->tls.sock.tx_sz = 0UL; + c->phase = PHASE_RESPONSE_SIGN; + *busy = 1; + return; + default: + break; + } + if( c->dialed ) { + FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` did not complete admission", + FD_IP4_ADDR_FMT_ARGS( c->address ) )); + } + drop( ch, idx, now ); +} + +/* Drive the candidate lifecycle state machine. */ +static void +service_candidate( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy, + uchar * payload ) { + struct candidate * c = &ch->candidates[idx]; + if( c->phase==PHASE_CONNECT ) { + if( connect_candidate( ch, idx, now ) ) *busy = 1; + return; + } + if( c->phasephase==PHASE_TLS ) { + if( !handshake_candidate( ch, idx, now, busy ) ) { + return; + } + } + if( flush( ch, idx, now, busy )!=0 ) return; + if( c->phase==PHASE_HELLO ) { + if( !read_candidate_hello( ch, idx, now, busy, payload ) ) { + return; + } + } + if( c->phase==PHASE_READY && !c->tx_used ) { + pair_candidate( ch, idx, now, busy ); + } +} + +/* The paired session went silent. */ +static int +silent( fd_failover_channel_t const * ch, long now ) { + long last_rx = ch->keepalive->ts_last_rx; + return now>=last_rx && now-last_rx>FD_FAILOVER_CHANNEL_IDLE_NANOS; +} + +/* Services the paired session. Returns 1 when a post-HELLO frame was delivered. */ +static int +service_paired( fd_failover_channel_t * ch, + long now, + int * busy, + ushort * type, + uchar * payload, + ulong * payload_sz ) { + ulong idx = (ulong)ch->paired_idx; + int delivered = 0; + int flushed = flush( ch, idx, now, busy ); + if( !flushed ) { + int rc = read_frame( ch, idx, now, busy, type, payload, payload_sz ); + if( rc>0 ) { + if( *type==FD_FAILOVER_MSG_HELLO ) { + lose( ch, idx, now, "it sent a second HELLO" ); + } + else { + fd_keepalive_rx( ch->keepalive, now ); + remember_source( ch, ch->candidates[idx].address, now ); + delivered = *type!=FD_FAILOVER_MSG_KEEPALIVE; + } + } else if( !rc && silent( ch, now ) ) { + lose( ch, idx, now, "it was idle past its deadline" ); + } + } + if( flushed>0 && silent( ch, now ) ) { + lose( ch, idx, now, "it stopped making progress" ); + } + if( ch->paired_idx>=0 && + !fd_failover_channel_tx_pending( ch ) && + fd_keepalive_should_tx( ch->keepalive, now ) ) + { + if( !fd_failover_channel_send( ch, now, FD_FAILOVER_MSG_KEEPALIVE, NULL, 0UL ) ) { + fd_keepalive_tx( ch->keepalive, ch->rng, now ); + *busy = 1; + } + } + return delivered; +} + +static void +dial( fd_failover_channel_t * ch, + long now, + int * busy ) { + if( !ch->dial_peer || !ch->peer_addr || ch->candidates[OUTBOUND_IDX].fd!=-1 || nowretry_at ) { + return; + } + + *busy = 1; + int fd = socket( AF_INET, SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC, 0 ); + if( fd==-1 ) { + drop( ch, OUTBOUND_IDX, now ); + return; + } + + int yes = 1; + if( setsockopt( fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes) ) ) { + close( fd ); + drop( ch, OUTBOUND_IDX, now ); + return; + } + + start( ch, OUTBOUND_IDX, fd, ch->peer_addr, now, PHASE_CONNECT ); +} + +int +fd_failover_channel_poll( fd_failover_channel_t * ch, + long now, + int * busy, + ushort * type, + uchar * payload, + ulong * payload_sz ) { + int delivered = 0; + ch->verify_budget = 1; + if( ch->paired_idx>=0 ) { + delivered = service_paired( ch, now, busy, type, payload, payload_sz ); + } + + /* Expire every slot each poll, independently of round-robin service. */ + for( ulong i=0; icandidates[i]; + if( c->fd!=-1 && (int)i!=ch->paired_idx && ( now>=c->deadline || now>=c->admission_until ) ) { + if( c->dialed ) { + FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` did not finish the handshake in time", + FD_IP4_ADDR_FMT_ARGS( c->address ) )); + } + drop( ch, i, now ); + *busy = 1; + } + } + + if( FD_UNLIKELY( !ch->tls_ctx.ready ) || ch->paired_idx>=0 ) { + return delivered; + } + dial( ch, now, busy ); + + ulong serviced = 0UL; + for( ulong idx=0UL; idxcandidates[idx].fd==-1 ) continue; + service_candidate( ch, idx, now, busy, payload ); + serviced++; + if( ch->paired_idx>=0 ) return delivered; + } + + for( ulong n=0UL; ncursor; + ch->cursor = (ch->cursor+1UL)%(FD_FAILOVER_CHANNEL_CANDIDATE_MAX-UNAUTHENTICATED_IDX); + if( ch->candidates[idx].fd==-1 ) continue; + if( has_proof( &ch->candidates[idx] ) && !ch->verify_budget ) continue; + + service_candidate( ch, idx, now, busy, payload ); + serviced++; + if( ch->paired_idx>=0 ) { + return delivered; + } + } + if( ch->listen_fd!=-1 ) { + accept_candidates( ch, now, busy ); + } + return delivered; +} + +int +fd_failover_channel_send( fd_failover_channel_t * ch, + long now, + ushort type, + uchar const * payload, + ulong sz ) { + if( ch->paired_idx<0 ) return -1; + + ulong idx = (ulong)ch->paired_idx; + struct candidate * c = &ch->candidates[idx]; + + if( c->tx_used ) return -1; + c->tx_used = fd_failover_wire_encode( c->tx, type, payload, sz ); + if( !c->tx_used ) { + lose( ch, idx, now, "we could not encode a frame for it" ); + return -1; + } + + int busy = 0; + int rc = flush( ch, idx, now, &busy )<0 ? -1 : 0; + if( !rc && type==FD_FAILOVER_MSG_HANDOFF_RESULT ) { + ch->expected_close = 1; + } + return rc; +} diff --git a/src/discof/failover/fd_failover_channel.h b/src/discof/failover/fd_failover_channel.h new file mode 100644 index 00000000000..1a6a4bdaed6 --- /dev/null +++ b/src/discof/failover/fd_failover_channel.h @@ -0,0 +1,147 @@ +#ifndef HEADER_fd_src_discof_failover_fd_failover_channel_h +#define HEADER_fd_src_discof_failover_fd_failover_channel_h + +/* Authenticated nonblocking TCP channel for the failover pair. */ + +#include "fd_failover_wire.h" +#include "../../waltz/tls/fd_tls.h" + +#include + +#define FD_FAILOVER_CHANNEL_MAGIC (0xF17EDA2CE5FA170AUL) + +/* Candidate slots */ +#define FD_FAILOVER_CHANNEL_CANDIDATE_MAX (16UL) + +#define FD_FAILOVER_NANOS_PER_SECOND (1000000000UL) +#define FD_FAILOVER_LOG_INTERVAL_NANOS (60L*(long)FD_FAILOVER_NANOS_PER_SECOND) + +/* Rate limits */ +#define FD_FAILOVER_ACCEPT_RATE (8192UL) /* number of num accept4 calls */ +#define FD_FAILOVER_ACCEPT_BURST (128UL) +#define FD_FAILOVER_ADMIT_RATE (128UL) /* number of candidates started */ +#define FD_FAILOVER_ADMIT_BURST (16UL) +#define FD_FAILOVER_VERIFY_RATE (128UL) /* number of Ed25519 proofs */ +#define FD_FAILOVER_VERIFY_BURST (16UL) +#define FD_FAILOVER_ACCEPT_PER_POLL (16UL) +#define FD_FAILOVER_LISTEN_BACKLOG (256) + +#define FD_FAILOVER_ADMISSION_START_NANOS ((long)FD_FAILOVER_NANOS_PER_SECOND) +#define FD_FAILOVER_ADMISSION_PROOF_NANOS (2L*(long)FD_FAILOVER_NANOS_PER_SECOND) + +struct fd_failover_proof_signer { + void * ctx; + void (* sign_fn)( void * ctx, uchar sig[ 64 ], uchar const msg[ FD_KEYGUARD_FAILOVER_PROOF_SZ ] ); +}; +typedef struct fd_failover_proof_signer fd_failover_proof_signer_t; + +/* Monotonic clock in nanoseconds. */ +static inline long +fd_failover_clock( void ) { + struct timespec ts; + clock_gettime( CLOCK_MONOTONIC, &ts ); + return (long)ts.tv_sec*(long)FD_FAILOVER_NANOS_PER_SECOND + (long)ts.tv_nsec; +} + +/* Handshake, operation and retry limits in nanoseconds. */ +#define FD_FAILOVER_CHANNEL_IDLE_NANOS ( 64000000000L) +#define FD_FAILOVER_CHANNEL_BACKOFF_MIN_NANOS ( 800000000L) +#define FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS ( 12800000000L) +#define FD_FAILOVER_CHANNEL_HELLO_TIMEOUT_NANOS ( 4000000000L) +#define FD_FAILOVER_CHANNEL_KEEPALIVE_NANOS (FD_FAILOVER_CHANNEL_IDLE_NANOS/4L) + +struct fd_failover_channel; +typedef struct fd_failover_channel fd_failover_channel_t; + +FD_PROTOTYPES_BEGIN + +FD_FN_CONST ulong +fd_failover_channel_align( void ); + +FD_FN_CONST ulong +fd_failover_channel_footprint( void ); + +void * +fd_failover_channel_new( void * shmem ); + +fd_failover_channel_t * +fd_failover_channel_join( void * shch ); + +/* Opens the listener on address:port. */ +void +fd_failover_channel_init_listener( fd_failover_channel_t * channel, + uint address, + ushort port ); + +/* Starts dialing and retrying at address:port. */ +void +fd_failover_channel_init_dialer( fd_failover_channel_t * channel, + uint address, + ushort port ); + +/* Configures admission and mutual TLS with separate proof and TLS signers. */ +int +fd_failover_channel_set_identity( fd_failover_channel_t * channel, + uchar const junk_pubkey[ 32 ], + fd_tls_sign_t signer, + fd_failover_proof_signer_t proof_signer, + fd_failover_hello_t const * hello ); + +/* Closes all sockets and releases the TLS context. */ +void +fd_failover_channel_fini( fd_failover_channel_t * channel ); + +/* Updates who holds the identity. */ +void +fd_failover_channel_set_holds( fd_failover_channel_t * channel, + int holds ); + +/* Drives the channel, now is the caller's clock in nanos. Sets + *charge_busy when it made progress. Returns 1 with *out_type, + out_payload and *out_payload_sz filled when a verified post HELLO frame + arrived, else 0. out_payload must hold FD_FAILOVER_PAYLOAD_MAX bytes. */ +int +fd_failover_channel_poll( fd_failover_channel_t * channel, + long now, + int * charge_busy, + ushort * out_type, + uchar * out_payload, + ulong * out_payload_sz ); + +/* Writes one authenticated frame, now is the caller's clock in nanos. + Returns 0 on success, -1 when the channel is not paired, already has a + pending frame, or encoding or writing fails. Encoding and write + failures tear down the session. */ +int +fd_failover_channel_send( fd_failover_channel_t * channel, + long now, + ushort type, + uchar const * payload, + ulong payload_sz ); + +/* Accessors. */ +FD_FN_PURE ulong fd_failover_channel_state ( fd_failover_channel_t const * channel ); +FD_FN_PURE int fd_failover_channel_listen_fd ( fd_failover_channel_t const * channel ); +FD_FN_PURE int fd_failover_channel_tx_pending( fd_failover_channel_t const * channel ); +FD_FN_PURE fd_failover_hello_t const * fd_failover_channel_peer_hello( fd_failover_channel_t const * channel ); + +/* Changes whenever a new authenticated session replaces the previous one. */ +FD_FN_PURE ulong fd_failover_channel_generation( fd_failover_channel_t const * channel ); + +/* Gets the bound port from the listen socket. */ +ushort +fd_failover_channel_listen_port( fd_failover_channel_t const * channel ); + +/* Closes the connection. */ +void +fd_failover_channel_hangup( fd_failover_channel_t * channel, + long now ); + +/* Drops the session after an error. */ +void +fd_failover_channel_protocol_error( fd_failover_channel_t * channel, + long now ); + +FD_PROTOTYPES_END + +#endif /* HEADER_fd_src_discof_failover_fd_failover_channel_h */ diff --git a/src/discof/failover/fd_failover_proto.c b/src/discof/failover/fd_failover_proto.c new file mode 100644 index 00000000000..6578815af96 --- /dev/null +++ b/src/discof/failover/fd_failover_proto.c @@ -0,0 +1,170 @@ +#include "fd_failover_proto.h" + +#include "../../ballet/ed25519/fd_ed25519.h" +#include "../../ballet/sha256/fd_sha256.h" +#include "../../disco/keyguard/fd_keyguard.h" + +FD_STATIC_ASSERT( sizeof(FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN)-1UL==FD_KEYGUARD_FAILOVER_DOMAIN_SZ, client_proof_domain ); +FD_STATIC_ASSERT( sizeof(FD_KEYGUARD_FAILOVER_SERVER_DOMAIN)-1UL==FD_KEYGUARD_FAILOVER_DOMAIN_SZ, server_proof_domain ); + +int +fd_failover_hello_check( fd_failover_hello_t const * self, + fd_failover_hello_t const * peer ) { + if( FD_UNLIKELY( self->version!=peer->version ) ) return FD_FAILOVER_HELLO_ERR_VERSION; + if( FD_UNLIKELY( self->holds>1 || peer->holds>1 ) ) return FD_FAILOVER_HELLO_ERR_HOLDS; + if( FD_UNLIKELY( self->mode!=FD_FAILOVER_MODE_ALPENGLOW || + self->mode!=peer->mode ) ) return FD_FAILOVER_HELLO_ERR_MODE; + if( FD_UNLIKELY( !self->boot_id || !peer->boot_id ) ) return FD_FAILOVER_HELLO_ERR_BOOT_ID; + if( FD_UNLIKELY( !fd_memeq( self->staked_pubkey, peer->staked_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_STAKED; + if( FD_UNLIKELY( !fd_memeq( self->vote_account, peer->vote_account, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_VOTE_ACCT; + if( FD_UNLIKELY( fd_memeq( self->junk_pubkey, peer->junk_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_JUNK_EQ; + if( FD_UNLIKELY( fd_memeq( self->junk_pubkey, self->staked_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_JUNK_STAKE; + if( FD_UNLIKELY( fd_memeq( peer->junk_pubkey, peer->staked_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_JUNK_STAKE; + if( FD_UNLIKELY( self->holds && peer->holds ) ) return FD_FAILOVER_HELLO_ERR_BOTH_HOLD; + return FD_FAILOVER_HELLO_OK; +} + +void +fd_failover_admission_hash( fd_failover_admission_start_t const * start, + uchar const * client_hello, + fd_failover_admission_challenge_t const * challenge, + uchar out[ 32 ] ) { + fd_sha256_t sha[1]; + fd_sha256_init( sha ); + fd_sha256_append( sha, start, sizeof(*start) ); + fd_sha256_append( sha, client_hello, start->client_hello_sz ); + fd_sha256_append( sha, challenge, sizeof(*challenge) ); + fd_sha256_fini( sha, out ); +} + +void +fd_failover_response_hash( uchar const transcript[ 32 ], + uchar const client_proof[ 64 ], + uchar const * server_flight, + ushort server_flight_sz, + uchar out[ 32 ] ) { + fd_sha256_t sha[1]; + fd_sha256_init( sha ); + fd_sha256_append( sha, transcript, 32UL ); + fd_sha256_append( sha, client_proof, 64UL ); + fd_sha256_append( sha, &server_flight_sz, sizeof(server_flight_sz) ); + fd_sha256_append( sha, server_flight, server_flight_sz ); + fd_sha256_fini( sha, out ); +} + +void +fd_failover_proof_msg( uchar out[ FD_KEYGUARD_FAILOVER_PROOF_SZ ], + int server, + uchar const junk_pubkey[ 32 ], + uchar const transcript[ 32 ] ) { + fd_memcpy( out, server ? FD_KEYGUARD_FAILOVER_SERVER_DOMAIN : FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN, + FD_KEYGUARD_FAILOVER_DOMAIN_SZ ); + fd_memcpy( out+FD_KEYGUARD_FAILOVER_DOMAIN_SZ, junk_pubkey, 32UL ); + fd_memcpy( out+FD_KEYGUARD_FAILOVER_DOMAIN_SZ+32UL, transcript, 32UL ); +} + +int +fd_failover_handoff_request_decode( fd_failover_handoff_request_t * out, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( payload_sz!=sizeof(fd_failover_handoff_request_t) ) ) return 0; + fd_failover_handoff_request_t request; + fd_memcpy( &request, payload, sizeof(request) ); + if( FD_UNLIKELY( !request.handoff_id || !request.target_boot_id ) ) return 0; + *out = request; + return 1; +} + +int +fd_failover_handoff_result_decode( fd_failover_handoff_result_t * out, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( payload_sz!=sizeof(fd_failover_handoff_result_t) ) ) return 0; + fd_failover_handoff_result_t result; + fd_memcpy( &result, payload, sizeof(result) ); + if( FD_UNLIKELY( !result.handoff_id ) ) return 0; + *out = result; + return 1; +} + +ulong +fd_failover_demoted_encode( uchar * out, + ulong handoff_id, + ulong target_boot_id, + ulong last_vote_slot, + uchar const * state, + ulong state_sz ) { + if( FD_UNLIKELY( !state_sz || state_sz>FD_FAILOVER_ALPENGLOW_STATE_MAX ) ) return 0UL; + + fd_failover_demoted_t demoted = { + .handoff_id = handoff_id, + .target_boot_id = target_boot_id, + .last_vote_slot = last_vote_slot, + .mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW, + .state_len = (ushort)state_sz, + }; + fd_memcpy( out, &demoted, sizeof(demoted) ); + fd_memcpy( out+sizeof(demoted), state, state_sz ); + return sizeof(demoted)+state_sz; +} + +int +fd_failover_demoted_decode( fd_failover_demoted_t * out, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( payload_szFD_FAILOVER_DEMOTED_PAYLOAD_MAX ) ) return 0; + + fd_failover_demoted_t demoted; + fd_memcpy( &demoted, payload, sizeof(demoted) ); + + ulong state_sz = payload_sz-sizeof(fd_failover_demoted_t); + /* The history itself is votor's to decode, the controller checks that + the adopted history ends at last_vote_slot. */ + if( FD_UNLIKELY( (ulong)demoted.state_len!=state_sz || + !state_sz || state_sz>FD_FAILOVER_ALPENGLOW_STATE_MAX || + demoted.mode!=(uchar)FD_FAILOVER_MODE_ALPENGLOW || + demoted.last_vote_slot==FD_FAILOVER_SLOT_NULL ) ) return 0; + + *out = demoted; + return 1; +} + +ulong +fd_failover_promote_ack_encode( uchar * out, + ulong handoff_id ) { + fd_failover_promote_ack_t ack = { .handoff_id=handoff_id }; + fd_memcpy( out, &ack, sizeof(ack) ); + return sizeof(ack); +} + +ulong +fd_failover_promote_rejected_encode( uchar * out, + ulong handoff_id, + uchar reason ) { + if( FD_UNLIKELY( reason==(uchar)FD_FAILOVER_REJECT_NONE || reason>=(uchar)FD_FAILOVER_REJECT_CNT ) ) return 0UL; + fd_failover_promote_rejected_t rej = { .handoff_id=handoff_id, .reason=reason }; + fd_memcpy( out, &rej, sizeof(rej) ); + return sizeof(rej); +} + +int +fd_failover_promote_ack_decode( fd_failover_promote_ack_t * out, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( payload_sz!=sizeof(fd_failover_promote_ack_t) ) ) return 0; + fd_memcpy( out, payload, sizeof(fd_failover_promote_ack_t) ); + return 1; +} + +int +fd_failover_promote_rejected_decode( fd_failover_promote_rejected_t * out, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( payload_sz!=sizeof(fd_failover_promote_rejected_t) ) ) return 0; + fd_failover_promote_rejected_t rej; + fd_memcpy( &rej, payload, sizeof(rej) ); + if( FD_UNLIKELY( rej.reason==(uchar)FD_FAILOVER_REJECT_NONE || rej.reason>=(uchar)FD_FAILOVER_REJECT_CNT ) ) return 0; + *out = rej; + return 1; +} diff --git a/src/discof/failover/fd_failover_proto.h b/src/discof/failover/fd_failover_proto.h new file mode 100644 index 00000000000..7473b5aa71c --- /dev/null +++ b/src/discof/failover/fd_failover_proto.h @@ -0,0 +1,233 @@ +#ifndef HEADER_fd_src_discof_failover_fd_failover_proto_h +#define HEADER_fd_src_discof_failover_fd_failover_proto_h + +/* Failover protocol messages and pure session transitions */ + +#include "../../util/fd_util_base.h" +#include "../../ballet/sha512/fd_sha512.h" +#include "../../disco/keyguard/fd_keyguard.h" + +/* Protocol version */ +#define FD_FAILOVER_VERSION (1U) + +/* The default port for a handoff. */ +#define FD_FAILOVER_DEFAULT_PORT ((ushort)8010) + +/* Message types */ +#define FD_FAILOVER_MSG_HELLO (0U) +#define FD_FAILOVER_MSG_HANDOFF_REQUEST (1U) +#define FD_FAILOVER_MSG_DEMOTED (2U) +#define FD_FAILOVER_MSG_PROMOTE_ACK (3U) +#define FD_FAILOVER_MSG_PROMOTE_REJECTED (4U) +#define FD_FAILOVER_MSG_HANDOFF_RESULT (5U) +#define FD_FAILOVER_MSG_KEEPALIVE (6U) +#define FD_FAILOVER_MSG_CNT (7U) + +/* Sentinel for a slot field with no value */ +#define FD_FAILOVER_SLOT_NULL (ULONG_MAX) + +/* Consensus payload formats */ +#define FD_FAILOVER_MODE_ALPENGLOW (1U) + +/* Session states for authenticated TCP sessions. */ +#define FD_FAILOVER_SESSION_LISTENING (0UL) /* Accept candidates until one authenticates. */ +#define FD_FAILOVER_SESSION_DIALING (1UL) /* Wait for outbound TCP to connect. */ +#define FD_FAILOVER_SESSION_HELLO (2UL) /* Complete TLS and exchange HELLO. */ +#define FD_FAILOVER_SESSION_PAIRED (3UL) /* Exchange messages with the authenticated peer. */ +#define FD_FAILOVER_SESSION_BACKOFF (4UL) /* Wait before the next dial attempt. */ +#define FD_FAILOVER_SESSION_CNT (5UL) + +/* HELLO pairing outcomes */ +#define FD_FAILOVER_HELLO_OK (0) +#define FD_FAILOVER_HELLO_ERR_VERSION (1) +#define FD_FAILOVER_HELLO_ERR_STAKED (2) +#define FD_FAILOVER_HELLO_ERR_VOTE_ACCT (3) +#define FD_FAILOVER_HELLO_ERR_JUNK_EQ (4) +#define FD_FAILOVER_HELLO_ERR_JUNK_STAKE (5) +#define FD_FAILOVER_HELLO_ERR_BOTH_HOLD (6) +#define FD_FAILOVER_HELLO_ERR_HOLDS (7) +#define FD_FAILOVER_HELLO_ERR_BOOT_ID (8) +#define FD_FAILOVER_HELLO_ERR_MODE (9) +#define FD_FAILOVER_HELLO_ERR_PROOF (10) + +/* Upper bound on the vote history payload in alpenglow mode, + AG_HIST_SER_MAX is 5274 bytes. */ +#define FD_FAILOVER_ALPENGLOW_STATE_MAX (6144UL) +#define FD_FAILOVER_STATE_MAX (FD_FAILOVER_ALPENGLOW_STATE_MAX) + +/* Wire protocol message bodies. Little endian, packed, fixed layout. */ +struct __attribute__((packed)) fd_failover_hello { + ushort version; /* FD_FAILOVER_VERSION */ + uchar junk_pubkey[ 32 ]; /* this host's junk key, generated at boot */ + uchar staked_pubkey[ 32 ]; /* the identity this pair hands off */ + uchar vote_account[ 32 ]; /* the vote account for this pair */ + uchar holds; /* 1 when this host runs staked_pubkey at pairing, else 0 */ + uchar mode; /* FD_FAILOVER_MODE_* consensus */ + ulong boot_id; /* random nonzero value per boot to distinguish restarts */ + uchar commit[ 20 ]; /* FD commit hash */ +}; +typedef struct fd_failover_hello fd_failover_hello_t; +FD_STATIC_ASSERT( sizeof(fd_failover_hello_t)==128UL, wire_layout ); + +/* Pre-TLS admission */ +#define FD_FAILOVER_ADMISSION_MAGIC (0x314d44414c494146UL) +#define FD_FAILOVER_ADMISSION_MAX (2048UL) +#define FD_FAILOVER_CLIENT_HELLO_MAX (1024UL) + +struct __attribute__((packed)) fd_failover_admission_start { + ulong magic; + fd_failover_hello_t hello; + uchar nonce[ 32 ]; + ushort client_hello_sz; +}; +typedef struct fd_failover_admission_start fd_failover_admission_start_t; +FD_STATIC_ASSERT( sizeof(fd_failover_admission_start_t)==170UL, admission_start_layout ); + +struct __attribute__((packed)) fd_failover_admission_challenge { + ulong magic; + fd_failover_hello_t hello; + uchar nonce[ 32 ]; +}; +typedef struct fd_failover_admission_challenge fd_failover_admission_challenge_t; +FD_STATIC_ASSERT( sizeof(fd_failover_admission_challenge_t)==168UL, admission_challenge_layout ); + +struct __attribute__((packed)) fd_failover_admission_response { + ushort server_flight_sz; + uchar proof[ 64 ]; +}; +typedef struct fd_failover_admission_response fd_failover_admission_response_t; +FD_STATIC_ASSERT( sizeof(fd_failover_admission_response_t)==66UL, admission_response_layout ); + +#define FD_FAILOVER_SERVER_FLIGHT_MAX (FD_FAILOVER_ADMISSION_MAX-sizeof(fd_failover_admission_response_t)) + +struct __attribute__((packed)) fd_failover_demoted { + ulong handoff_id; /* the handoff id from the HANDOFF_REQUEST */ + ulong target_boot_id; /* peer boot_id the handoff is for */ + ulong last_vote_slot; /* tip of the final history */ + uchar mode; /* FD_FAILOVER_MODE_* encoding */ + ushort state_len; /* size of the final history that follows this struct */ +}; +typedef struct fd_failover_demoted fd_failover_demoted_t; +FD_STATIC_ASSERT( sizeof(fd_failover_demoted_t)==27UL, wire_layout ); + +#define FD_FAILOVER_DEMOTED_PAYLOAD_MAX (sizeof(fd_failover_demoted_t)+FD_FAILOVER_STATE_MAX) + +struct __attribute__((packed)) fd_failover_promote_ack { + ulong handoff_id; +}; +typedef struct fd_failover_promote_ack fd_failover_promote_ack_t; +FD_STATIC_ASSERT( sizeof(fd_failover_promote_ack_t)==8UL, wire_layout ); + +struct __attribute__((packed)) fd_failover_promote_rejected { + ulong handoff_id; + uchar reason; /* FD_FAILOVER_REJECT_* */ +}; +typedef struct fd_failover_promote_rejected fd_failover_promote_rejected_t; +FD_STATIC_ASSERT( sizeof(fd_failover_promote_rejected_t)==9UL, wire_layout ); + +struct __attribute__((packed)) fd_failover_handoff_request { + ulong handoff_id; + ulong target_boot_id; + ulong replay_slot; /* replay slot of the machine asking for the identity, FD_FAILOVER_SLOT_NULL before replay starts */ +}; +typedef struct fd_failover_handoff_request fd_failover_handoff_request_t; +FD_STATIC_ASSERT( sizeof(fd_failover_handoff_request_t)==24UL, wire_layout ); + +/* Ends the exchange, result is an admin control result. */ +struct __attribute__((packed)) fd_failover_handoff_result { + ulong handoff_id; + ulong result; +}; +typedef struct fd_failover_handoff_result fd_failover_handoff_result_t; +FD_STATIC_ASSERT( sizeof(fd_failover_handoff_result_t)==16UL, wire_layout ); + +/* PROMOTE_REJECTED reasons */ +#define FD_FAILOVER_REJECT_NONE (0U) +#define FD_FAILOVER_REJECT_BUSY (1U) +#define FD_FAILOVER_REJECT_HOLDS_IDENTITY (2U) +#define FD_FAILOVER_REJECT_REPLAY_BEHIND (3U) +#define FD_FAILOVER_REJECT_ADOPTION_MISMATCH (4U) +#define FD_FAILOVER_REJECT_ADOPTION_FAILED (5U) +#define FD_FAILOVER_REJECT_SWITCH_FAILED (6U) +#define FD_FAILOVER_REJECT_CNT (7U) + +FD_PROTOTYPES_BEGIN + +/* Checks a peer's HELLO against ours and returns FD_FAILOVER_HELLO_OK or + the first fatal error. The checks are symmetric on both nodes. */ +int +fd_failover_hello_check( fd_failover_hello_t const * self, + fd_failover_hello_t const * peer ); + +/* Admission hash covers exact packed wire bytes and the prepared TLS + record. response_hash binds the listener proof to the client proof + and the server's prepared TLS flight, including its ephemeral key. */ +void +fd_failover_admission_hash( fd_failover_admission_start_t const * start, + uchar const * client_hello, + fd_failover_admission_challenge_t const * challenge, + uchar out[ 32 ] ); + +void +fd_failover_response_hash( uchar const transcript[ 32 ], + uchar const client_proof[ 64 ], + uchar const * server_flight, + ushort server_flight_sz, + uchar out[ 32 ] ); + +void +fd_failover_proof_msg( uchar out[ FD_KEYGUARD_FAILOVER_PROOF_SZ ], + int server, + uchar const junk_pubkey[ 32 ], + uchar const transcript[ 32 ] ); + +/* Validate a handoff request or response. */ +int +fd_failover_handoff_request_decode( fd_failover_handoff_request_t * out, + uchar const * payload, + ulong payload_sz ); + +int +fd_failover_handoff_result_decode( fd_failover_handoff_result_t * out, + uchar const * payload, + ulong payload_sz ); + +/* Writes a DEMOTED payload. */ +ulong +fd_failover_demoted_encode( uchar * out, + ulong handoff_id, + ulong target_boot_id, + ulong last_vote_slot, + uchar const * state, + ulong state_sz ); + +/* Validates a DEMOTED header. */ +int +fd_failover_demoted_decode( fd_failover_demoted_t * out, + uchar const * payload, + ulong payload_sz ); + +/* Write a PROMOTE_ACK or PROMOTE_REJECTED payload. */ +ulong +fd_failover_promote_ack_encode( uchar * out, + ulong handoff_id ); + +ulong +fd_failover_promote_rejected_encode( uchar * out, + ulong handoff_id, + uchar reason ); + +/* Validate a PROMOTE_ACK or PROMOTE_REJECTED payload. */ +int +fd_failover_promote_ack_decode( fd_failover_promote_ack_t * out, + uchar const * payload, + ulong payload_sz ); + +int +fd_failover_promote_rejected_decode( fd_failover_promote_rejected_t * out, + uchar const * payload, + ulong payload_sz ); + +FD_PROTOTYPES_END + +#endif /* HEADER_fd_src_discof_failover_fd_failover_proto_h */ diff --git a/src/discof/failover/fd_failover_tile.c b/src/discof/failover/fd_failover_tile.c new file mode 100644 index 00000000000..71e3c0feee7 --- /dev/null +++ b/src/discof/failover/fd_failover_tile.c @@ -0,0 +1,2670 @@ +#include "../../disco/topo/fd_topo.h" +#include "../../disco/topo/fd_dns_resolve.h" +#include "../../disco/keyguard/fd_keyload.h" +#include "../../disco/keyguard/fd_keyguard.h" +#include "../../disco/keyguard/fd_keyguard_client.h" +#include "../../flamenco/gossip/fd_gossip_message.h" +#include "../../ballet/base58/fd_base58.h" +#include "../../ballet/hex/fd_hex.h" +#include "../../util/fd_version.h" +#include "../../util/net/fd_ip4.h" +#include "../votor/fd_votor_tile.h" + +#include "fd_failover_bus.h" +#include "fd_failover_channel.h" + +#include +#include +#include /* IPPROTO_TCP (seccomp) */ +#include /* TCP_NODELAY (seccomp) */ + +#include "generated/fd_failover_tile_seccomp.h" + +/* The failov tile owns the sockets to the other failover machine, keeps + host networking for them and holds no private keys. The sign tile + signs fresh admission proofs with the failover identity and our TLS + handshakes with the junk key, and the admin tile does the identity + switch and stays off the network. */ + +/* Idle socket poll spacing */ +#define FD_FAILOVER_TILE_POLL_NANOS (50000L) /* 50us */ + +/* A transition gives up once replay passes this many slots past its + start. */ +#define FD_FAILOVER_DEADLINE_SLOTS (64UL) + +/* The promotion's replay and adopt waits also end on our clock, one + second per deadline slot, so a machine whose replay froze eventually + stops. */ +#define FD_FAILOVER_DEADLINE_SLOT_NANOS ((long)FD_FAILOVER_NANOS_PER_SECOND) + +/* A contact info for the identity from another host newer than this + means that host runs it, two gossip refreshes of 7.5s. */ +#define FD_FAILOVER_GOSSIP_FRESH_NANOS (15000000000L) +#define FD_FAILOVER_REQUEST_RETRIES (3UL) /* sends of one request before a missing final history ends it */ + +/* The controller actions, promotion sources and handoff results are + in fd_adminctl.h, `failover status` reports them. */ + +/* Which key a switch installs, CNT if no switch is in flight */ +#define FD_FAILOVER_SWITCH_KEY_JUNK (0UL) /* The junk identity. */ +#define FD_FAILOVER_SWITCH_KEY_STAKED (1UL) /* The staked identity. */ +#define FD_FAILOVER_SWITCH_KEY_CNT (2UL) /* No identity switch is pending. */ + +/* Our DEMOTED to the handoff target */ +#define FD_FAILOVER_DELIVERY_NONE (0UL) /* Nothing to send. */ +#define FD_FAILOVER_DELIVERY_PENDING (1UL) /* Not sent on this session yet. */ +#define FD_FAILOVER_DELIVERY_SENT (2UL) /* Sent on this session, a new session sends it again. */ + +/* Our last response to a DEMOTED */ +#define FD_FAILOVER_REPLY_NONE (0UL) /* No response yet. */ +#define FD_FAILOVER_REPLY_KEPT (1UL) /* Kept, the same DEMOTED again gets it again. */ +#define FD_FAILOVER_REPLY_PENDING (2UL) /* Kept, not queued yet (pending) because the control slot was busy. */ + +/* A serialized Alpenglow vote history and its tip. */ +struct fd_failover_history { + int valid; /* holds a history */ + ulong tip; /* slot of its last vote */ + ulong sz; /* bytes used in state */ + uchar state[ FD_FAILOVER_STATE_MAX ]; /* the serialized history */ +}; + +typedef struct fd_failover_history fd_failover_history_t; + +/* The member and boot a message or request is bound to. */ +struct fd_failover_peer { + ulong boot_id; /* the peer's boot */ + uchar junk[ 32 ]; /* the peer's junk key */ +}; + +typedef struct fd_failover_peer fd_failover_peer_t; + +static inline int +peer_eq( fd_failover_peer_t const * bound, + ulong boot_id, + uchar const * junk ) { + return bound->boot_id==boot_id && fd_memeq( bound->junk, junk, 32UL ); +} + +/* The session we last saw on the channel. */ +struct fd_failover_session { + ulong state; /* last seen session state */ + ulong generation; /* catches a replacement paired during one poll */ + ulong peer_boot_id; /* boot of the peer on this session */ + int peer_holds; /* the peer runs the identity, from its HELLO or an accepted handoff on this session */ + int close_after_send; /* hang up once the queued message is sent */ + ulong close_handoff_id; /* the handoff that hang up is for, for the log */ +}; + +typedef struct fd_failover_session fd_failover_session_t; + +/* Our handoff request, sent by the machine that wants the identity. */ +struct fd_failover_request { + ulong id; /* zero when none is open */ + fd_failover_peer_t peer; /* the boot we authenticated for it, which runs the identity */ + uint addr; /* the address we dial, a resume may replace it */ + long until; /* when the request pauses, zero while paused */ + int sent; /* sent on this session */ + int peer_restarted; /* paused because the authenticated member or boot changed */ + ulong last_id; /* id of our latest request, kept for status */ + ulong result; /* FD_FAILOVER_HANDOFF_* of our latest request */ + ulong send_cnt; /* how many times we sent it */ + long log_at; /* when we may log a resend again */ +}; + +typedef struct fd_failover_request fd_failover_request_t; + +/* Our demotion. */ +struct fd_failover_handoff { + ulong id; /* id of our latest demotion */ + fd_failover_peer_t target; /* peer boot it is for */ + ulong result; /* FD_FAILOVER_HANDOFF_* of the latest DEMOTED */ + ulong delivery; /* FD_FAILOVER_DELIVERY_* */ + long until; /* the demotion step's deadline on our clock, for when replay stops */ + long log_at; /* when we may log a resend again */ + long result_log_at; /* when we may log the result again */ + ulong payload_sz; /* bytes used in payload */ + uchar payload[ FD_FAILOVER_DEMOTED_PAYLOAD_MAX ]; /* our DEMOTED, resent on a new session */ +}; + +typedef struct fd_failover_handoff fd_failover_handoff_t; + +/* The promotion in progress. */ +struct fd_failover_promotion { + char label[ 48 ]; /* handoff ID or operator promotion, fixed for every step */ + ulong source; /* FD_FAILOVER_SOURCE_* we adopt from */ + int from_peer; /* the peer's DEMOTED requested it, we send the response */ + fd_failover_peer_t peer; /* that DEMOTED's member boot, it gets our response */ + ulong handoff_id; /* and its handoff id */ + ulong floor; /* coverage floor when it started */ + int force; /* `failover promote --force`, we skip the peer checks and accept incomplete or empty history */ + int holder_seen; /* a peer running the identity authenticated after promotion started */ + long until; /* our clock when the replay and adopt waits give up, 0 until set */ + fd_failover_history_t adopt; /* history or the bound sent to votor */ + ulong adopt_id; /* id of the adoption we asked the votor for */ + int retry; /* ask the votor again once replay passes retry_slot */ + ulong retry_slot; /* replay slot that retry waits for */ + int floor_logged; /* we said once that we wait for the floor */ + int retry_logged; /* and that we ask the votor again as replay moves */ +}; + +typedef struct fd_failover_promotion fd_failover_promotion_t; + +/* Our last response to a DEMOTED. The same DEMOTED again gets the same + response, so a peer that missed it is not left waiting. */ +struct fd_failover_reply { + ulong state; /* FD_FAILOVER_REPLY_* */ + fd_failover_peer_t peer; /* the member boot it goes to */ + ulong handoff_id; /* the handoff it answers */ + ushort type; /* PROMOTE_ACK or PROMOTE_REJECTED */ + uchar reason; /* for a rejected reply */ + long log_at; /* when we may log a resend again */ +}; + +typedef struct fd_failover_reply fd_failover_reply_t; + +/* The one control message waiting to go out, if any. */ +struct fd_failover_tx { + ushort type; /* FD_FAILOVER_MSG_* */ + ushort sz; /* bytes used in payload */ + int valid; /* a message is waiting */ + fd_failover_peer_t to; /* the member boot it goes to */ + uchar payload[ FD_FAILOVER_DEMOTED_PAYLOAD_MAX ]; /* the encoded message */ +}; + +typedef struct fd_failover_tx fd_failover_tx_t; + +/* State of the identity switch we asked the admin tile for. It + selects a key preloaded by the signing tile by its public key, and + success means the old key is gone from every tile's installed identity. */ +struct fd_failover_id_switch { + ulong request_id; /* nonce of our latest switch request */ + ulong pending_key; /* FD_FAILOVER_SWITCH_KEY_*, or CNT when idle */ + fd_failover_switch_resp_t result; /* the committed response */ + fd_failover_switch_resp_t response; /* uncommitted input frame */ + ulong response_nonce; /* nonce of the frame being consumed */ + int fresh; /* a response frame arrived and is not handled yet */ + int overdue; /* the switch in flight passed its deadline, we still wait for it */ + int discard; /* set-identity ran while it was in flight, its answer is dropped */ + ulong epoch; /* failover epoch we last saw, sent with every request */ +}; + +typedef struct fd_failover_id_switch fd_failover_id_switch_t; + +struct fd_failover_tile_ctx { + fd_keyguard_client_t keyguard_client[ 1 ]; /* signs admission proofs and TLS */ + int input_lost; /* input overrun disables failover until restart */ + int read_pending; /* cleared only after STEM validates the fragment */ + fd_failover_hello_t hello; /* the HELLO we send */ + int holds; /* we run the target identity, hello.staked_pubkey */ + uchar installed[ 32 ]; /* the identity this machine signs with now, from the switches we asked for and set-identity */ + uint cmd_addr; /* from `failover promote --address`, 0 when gossip finds the identity */ + ushort cmd_port; /* from `failover promote --port`, 0 for the default port */ + fd_failover_channel_t * channel; /* the TLS channel to the peer */ + fd_failover_session_t session; /* the session we last saw on the channel */ + long poll_at; /* next idle socket poll */ + + fd_failover_request_t request; /* our handoff request */ + int last_requested; /* status describes our request rather than our demotion */ + long handoff_refuse_log_at; /* when we may log a refused handoff request again */ + + ulong peer_floor; /* highest last vote the peer reported for the target, kept across sessions */ + ulong own_floor; /* highest vote this machine signed as the target */ + + /* gossip_ciseen, contact info updates */ + ulong gossip_in_idx; /* gossip_ciseen in link */ + fd_wksp_t * gossip_in_mem; /* its workspace */ + ulong gossip_in_chunk0; /* its first chunk */ + ulong gossip_in_wmark; /* its last chunk */ + ulong gossip_in_mtu; /* its largest frag */ + + fd_ip4_port_t own_gossip; /* our own gossip socket */ + uchar gossip_origin[ 32 ]; /* copied in during_frag */ + fd_ip4_port_t gossip_socket; /* copied in during_frag, zero for a remove or IPv6 */ + uint staked_addr; /* from a staked contact info that is not ours, 0 none */ + long staked_seen_at; /* our clock when that one arrived, 0 never */ + + /* Controller */ + ulong action; /* FD_FAILOVER_ACTION_* */ + int stuck; /* a transition failed or is overdue, shown to the operator */ + int stuck_logged; /* the stuck value we last told the operator about */ + ulong deadline_slot; /* replay slot past which the current step is overdue */ + + /* Handoff ids count up from a random base per boot, so ids from + different boots of ours do not collide. */ + ulong handoff_base; /* random base for our handoff ids */ + ulong handoff_cnt; /* handoff ids issued since boot */ + fd_failover_handoff_t handoff; /* our demotion */ + int taken; /* the peer took the identity, our next tenure or a set-identity clears this */ + + fd_failover_history_t peer_history; /* stored peer history, from a DEMOTED */ + + fd_failover_promotion_t promotion; /* the promotion in progress */ + fd_failover_reply_t reply; /* our last response to a DEMOTED */ + fd_failover_tx_t tx; /* the control message waiting to go out */ + + /* The admin tile forwards operator commands over the bus and waits for + one response frame per request. */ + ulong admin_in_idx; /* admin_failov in link */ + fd_wksp_t * admin_in_mem; /* its workspace */ + ulong admin_in_chunk0; /* its first chunk */ + ulong admin_in_wmark; /* its last chunk */ + ulong admin_out_idx; /* failov_admin out link */ + fd_wksp_t * admin_out_mem; /* its workspace */ + ulong admin_out_chunk0; /* its first chunk */ + ulong admin_out_wmark; /* its last chunk */ + ulong admin_out_chunk; /* next chunk we publish to */ + fd_failover_bus_msg_t bus_req; /* the operator command copied in during_frag */ + int bus_req_fresh; /* bus_req arrived and is not handled yet */ + fd_failover_operator_t bus_operator; /* copied in during_frag */ + + fd_failover_id_switch_t id_switch; /* the identity switch we asked the admin tile for */ + + /* Adoption request to the votor and its response. */ + ulong adopt_out_idx; /* adoption request out link */ + fd_wksp_t * adopt_out_mem; /* its workspace */ + ulong adopt_out_chunk0; /* its first chunk */ + ulong adopt_out_wmark; /* its last chunk */ + ulong adopt_out_chunk; /* next chunk we publish to */ + ulong adopt_request_id; /* id of our latest adoption request */ + ulong adopt_in_idx; /* adoption response in link */ + fd_wksp_t * adopt_in_mem; /* its workspace */ + ulong adopt_in_chunk0; /* its first chunk */ + ulong adopt_in_wmark; /* its last chunk */ + fd_votor_adopt_result_t adopt_result; /* the adoption response copied in during_frag */ + ulong adopt_result_id; /* the request it answers */ + int adopt_result_fresh; /* adopt_result arrived and is not handled yet */ + + /* Vote history input. */ + ulong hist_in_idx; /* votor_hist in link */ + fd_wksp_t * hist_in_mem; /* its workspace */ + ulong hist_in_chunk0; /* its first chunk */ + ulong hist_in_wmark; /* its last chunk */ + + int hist_fresh; /* a history frame arrived and is not handled yet */ + ulong hist_seen_seq; /* seq of the last votor_hist frag taken in, ULONG_MAX before any */ + + /* Votor history and adoption bounds. */ + fd_votor_hist_msg_t hist; /* the votor_hist frame copied in during_frag */ + ulong adopt_anchor; /* finality anchor of the history being adopted, SLOT_NULL when none */ + ulong empty_vote_after; /* an empty history votes only past this slot, SLOT_NULL until known */ + int hist_warned; /* we warned of a bad frame and no good one came since */ + + ulong replay_slot; /* latest replayed slot */ + ulong root_slot; /* latest root */ + ulong last_vote_slot; /* our latest vote */ + + fd_failover_history_t current_history; /* history of our latest vote */ + + uchar rx[ FD_FAILOVER_PAYLOAD_MAX ]; /* the control message being decoded */ +}; + +typedef struct fd_failover_tile_ctx fd_failover_tile_ctx_t; + +FD_FN_CONST static inline ulong +scratch_align( void ) { + return fd_ulong_max( alignof(fd_failover_tile_ctx_t), fd_failover_channel_align() ); +} + +FD_FN_PURE static inline ulong +scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) { + ulong l = FD_LAYOUT_INIT; + l = FD_LAYOUT_APPEND( l, alignof(fd_failover_tile_ctx_t), sizeof(fd_failover_tile_ctx_t) ); + l = FD_LAYOUT_APPEND( l, fd_failover_channel_align(), fd_failover_channel_footprint() ); + return FD_LAYOUT_FINI( l, scratch_align() ); +} + +/* TLS CertificateVerify, signed by the sign tile with the junk key. */ +static void +sign_ed25519( void * signer_ctx, + uchar sig[ static FD_ED25519_SIG_SZ ], + uchar const msg[ static FD_TLS_CV_SIGN_SZ ] ) { + fd_failover_tile_ctx_t * ctx = signer_ctx; + fd_keyguard_client_sign( ctx->keyguard_client, sig, msg, FD_TLS_CV_SIGN_SZ, FD_KEYGUARD_SIGN_TYPE_ED25519 ); +} + +static void +sign_proof( void * signer_ctx, + uchar sig[ 64 ], + uchar const msg[ FD_KEYGUARD_FAILOVER_PROOF_SZ ] ) { + fd_failover_tile_ctx_t * ctx = signer_ctx; + fd_keyguard_client_sign( ctx->keyguard_client, sig, msg, FD_KEYGUARD_FAILOVER_PROOF_SZ, FD_KEYGUARD_SIGN_TYPE_ED25519 ); +} + +static void +privileged_init( fd_topo_t const * topo, + fd_topo_tile_t const * tile ) { + void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id ); + FD_SCRATCH_ALLOC_INIT( l, scratch ); + fd_failover_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_failover_tile_ctx_t), sizeof(fd_failover_tile_ctx_t) ); + void * channel_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_failover_channel_align(), fd_failover_channel_footprint() ); + /* Start idle with no handoff or promotion in progress. */ + fd_memset( ctx, 0, sizeof(fd_failover_tile_ctx_t) ); + /* No admin switch is pending at boot. */ + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT; + ctx->deadline_slot = FD_FAILOVER_SLOT_NULL; + ctx->peer_floor = FD_FAILOVER_SLOT_NULL; + ctx->own_floor = FD_FAILOVER_SLOT_NULL; + ctx->replay_slot = FD_FAILOVER_SLOT_NULL; + ctx->root_slot = FD_FAILOVER_SLOT_NULL; + ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; + ctx->hist_seen_seq = ULONG_MAX; + FD_TEST( fd_rng_secure( &ctx->handoff_base, sizeof(ulong) ) ); + + /* The junk key generated at boot goes into HELLO and TLS, and a + handoff switches to it. Only Sign holds its private key. The + configured identity is what we hand off until set-identity or + failover promote switches to another key. */ + uchar const * junk_pubkey = fd_keyload_load( FD_KEYLOAD_FAILOVER_JUNK, 1 ); + fd_memcpy( ctx->hello.junk_pubkey, junk_pubkey, 32UL ); + fd_keyload_unload( junk_pubkey, 1 ); + uchar const * target_pubkey = fd_keyload_load( tile->failov.identity_key_path, 1 ); + fd_memcpy( ctx->hello.staked_pubkey, target_pubkey, 32UL ); + fd_memcpy( ctx->installed, tile->failov.boot_to_standby ? ctx->hello.junk_pubkey : target_pubkey, 32UL ); + fd_keyload_unload( target_pubkey, 1 ); + + if( FD_UNLIKELY( fd_memeq( ctx->hello.junk_pubkey, ctx->hello.staked_pubkey, 32UL ) ) ) { + FD_LOG_ERR(( "the failover junk key must differ from [paths.identity_key]" )); + } + + /* The vote key is a base58 pubkey or a keypair file, same as in the + votor. */ + if( FD_UNLIKELY( !fd_base58_decode_32( tile->failov.vote_account_path, ctx->hello.vote_account ) ) ) { + if( FD_UNLIKELY( !strcmp( tile->failov.vote_account_path, "" ) ) ) FD_LOG_ERR(( "missing [paths.vote_account]" )); + uchar const * vote_account = fd_keyload_load( tile->failov.vote_account_path, 1 ); + fd_memcpy( ctx->hello.vote_account, vote_account, 32UL ); + fd_keyload_unload( vote_account, 1 ); + } + + ctx->adopt_anchor = FD_FAILOVER_SLOT_NULL; + ctx->empty_vote_after = FD_FAILOVER_SLOT_NULL; + + /* We run the target identity unless we booted to standby. */ + ctx->holds = !tile->failov.boot_to_standby; + ctx->hello.version = (ushort)FD_FAILOVER_VERSION; + ctx->hello.holds = (uchar)ctx->holds; + ctx->hello.mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; + while( FD_UNLIKELY( !ctx->hello.boot_id ) ) FD_TEST( fd_rng_secure( &ctx->hello.boot_id, sizeof(ulong) ) ); + /* The commit field has the hash bytes, a build without one leaves it + zero. */ + if( FD_LIKELY( strlen( fd_commit_ref_cstr )==2UL*sizeof(ctx->hello.commit) && + fd_hex_decode( ctx->hello.commit, fd_commit_ref_cstr, sizeof(ctx->hello.commit) )!=sizeof(ctx->hello.commit) ) ) { + fd_memset( ctx->hello.commit, 0, sizeof(ctx->hello.commit) ); + } + + ctx->own_gossip = tile->failov.gossip_addr; + if( FD_UNLIKELY( tile->failov.gossip_host[ 0 ]!='\0' ) ) { + if( FD_UNLIKELY( !fd_dns_resolve_address( tile->failov.gossip_host, &ctx->own_gossip.addr ) ) ) { + FD_LOG_ERR(( "could not resolve [gossip.host] `%s`", tile->failov.gossip_host )); + } + } + + ctx->channel = fd_failover_channel_join( fd_failover_channel_new( channel_mem ) ); + FD_TEST( ctx->channel ); + fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 }; + if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, (fd_failover_proof_signer_t){ .ctx=ctx, .sign_fn=sign_proof }, &ctx->hello ) ) ) { + FD_LOG_ERR(( "failover TLS setup failed for the junk key" )); + } + + fd_failover_channel_init_listener( ctx->channel, tile->failov.listen_addr, tile->failov.port ); + ctx->session.state = fd_failover_channel_state( ctx->channel ); + + char junk_b58 [ FD_BASE58_ENCODED_32_SZ ]; + char staked_b58[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( ctx->hello.junk_pubkey, NULL, junk_b58 ); + fd_base58_encode_32( ctx->hello.staked_pubkey, NULL, staked_b58 ); + if( FD_UNLIKELY( tile->failov.boot_to_standby ) ) { + FD_LOG_NOTICE(( "failover is on, booting as a standby under junk key `%s` for identity `%s`, listening on `" FD_IP4_ADDR_FMT ":%u`, " + "this machine does not vote until `failover promote` here takes the identity from the machine running it.", + junk_b58, staked_b58, FD_IP4_ADDR_FMT_ARGS( tile->failov.listen_addr ), (uint)tile->failov.port )); + } else { + FD_LOG_NOTICE(( "failover is on, listening on `" FD_IP4_ADDR_FMT ":%u` with junk key `%s`, a handoff from this machine gives up `%s` for the junk key, " + "`failover promote ` here takes the identity in that keypair from the machine running it", + FD_IP4_ADDR_FMT_ARGS( tile->failov.listen_addr ), (uint)tile->failov.port, junk_b58, staked_b58 )); + } + + FD_TEST( FD_SCRATCH_ALLOC_FINI( l, scratch_align() )==(ulong)scratch+scratch_footprint( tile ) ); +} + +static void +unprivileged_init( fd_topo_t const * topo, + fd_topo_tile_t const * tile ) { + fd_failover_tile_ctx_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id ); + + ctx->gossip_in_idx = ULONG_MAX; + ctx->hist_in_idx = ULONG_MAX; + ctx->admin_in_idx = ULONG_MAX; + ctx->adopt_in_idx = ULONG_MAX; + ctx->adopt_out_idx = fd_topo_find_tile_out_link( topo, tile, "failov_votor", 0UL ); + if( FD_LIKELY( ctx->adopt_out_idx!=ULONG_MAX ) ) { + fd_topo_link_t const * link = &topo->links[ tile->out_link_id[ ctx->adopt_out_idx ] ]; + ctx->adopt_out_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->adopt_out_chunk0 = fd_dcache_compact_chunk0( ctx->adopt_out_mem, link->dcache ); + ctx->adopt_out_wmark = fd_dcache_compact_wmark ( ctx->adopt_out_mem, link->dcache, link->mtu ); + ctx->adopt_out_chunk = ctx->adopt_out_chunk0; + } + ctx->admin_out_idx = fd_topo_find_tile_out_link( topo, tile, "failov_admin", 0UL ); + if( FD_LIKELY( ctx->admin_out_idx!=ULONG_MAX ) ) { + fd_topo_link_t const * link = &topo->links[ tile->out_link_id[ ctx->admin_out_idx ] ]; + ctx->admin_out_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->admin_out_chunk0 = fd_dcache_compact_chunk0( ctx->admin_out_mem, link->dcache ); + ctx->admin_out_wmark = fd_dcache_compact_wmark ( ctx->admin_out_mem, link->dcache, link->mtu ); + ctx->admin_out_chunk = ctx->admin_out_chunk0; + } + for( ulong i=0UL; iin_cnt; i++ ) { + fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ]; + if( FD_LIKELY( !strcmp( link->name, "gossip_ciseen" ) ) ) { + ctx->gossip_in_idx = i; + ctx->gossip_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->gossip_in_chunk0 = fd_dcache_compact_chunk0( ctx->gossip_in_mem, link->dcache ); + ctx->gossip_in_wmark = fd_dcache_compact_wmark ( ctx->gossip_in_mem, link->dcache, link->mtu ); + ctx->gossip_in_mtu = link->mtu; + continue; + } + if( FD_LIKELY( !strcmp( link->name, "votor_failov" ) ) ) { + ctx->adopt_in_idx = i; + ctx->adopt_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->adopt_in_chunk0 = fd_dcache_compact_chunk0( ctx->adopt_in_mem, link->dcache ); + ctx->adopt_in_wmark = fd_dcache_compact_wmark ( ctx->adopt_in_mem, link->dcache, link->mtu ); + continue; + } + if( FD_LIKELY( !strcmp( link->name, "admin_failov" ) ) ) { + ctx->admin_in_idx = i; + ctx->admin_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->admin_in_chunk0 = fd_dcache_compact_chunk0( ctx->admin_in_mem, link->dcache ); + ctx->admin_in_wmark = fd_dcache_compact_wmark ( ctx->admin_in_mem, link->dcache, link->mtu ); + continue; + } + if( FD_LIKELY( !strcmp( link->name, "votor_hist" ) ) ) { + ctx->hist_in_idx = i; + ctx->hist_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->hist_in_chunk0 = fd_dcache_compact_chunk0( ctx->hist_in_mem, link->dcache ); + ctx->hist_in_wmark = fd_dcache_compact_wmark ( ctx->hist_in_mem, link->dcache, link->mtu ); + continue; + } + /* The keyguard client reads sign_failov itself. */ + if( FD_LIKELY( !strcmp( link->name, "sign_failov" ) ) ) continue; + FD_LOG_ERR(( "unexpected input link name %s", link->name )); + } + + ulong sign_in_idx = fd_topo_find_tile_in_link ( topo, tile, "sign_failov", tile->kind_id ); + ulong sign_out_idx = fd_topo_find_tile_out_link( topo, tile, "failov_sign", tile->kind_id ); + FD_TEST( sign_in_idx!=ULONG_MAX && sign_out_idx!=ULONG_MAX ); + fd_topo_link_t const * sign_in = &topo->links[ tile->in_link_id [ sign_in_idx ] ]; + fd_topo_link_t const * sign_out = &topo->links[ tile->out_link_id[ sign_out_idx ] ]; + + fd_sleep_t * sleep = NULL; + if( FD_UNLIKELY( topo->sleep_obj_id!=ULONG_MAX ) ) { + sleep = fd_sleep_join( fd_topo_obj_laddr( topo, topo->sleep_obj_id ) ); + FD_TEST( sleep ); + } + + if( FD_UNLIKELY( !fd_keyguard_client_join( fd_keyguard_client_new( ctx->keyguard_client, + sign_out->mcache, + sign_out->dcache, + sign_in->mcache, + sign_in->dcache, + sign_out->mtu, + sign_in->mtu, + sleep, + sign_out->id, + fd_topo_find_link_consumer( topo, sign_out ) ) ) ) ) { + FD_LOG_ERR(( "failed to construct keyguard" )); + } +} + +/* Updates our session state when the channel's session changes. */ +static void +sync_session( fd_failover_tile_ctx_t * ctx ) { + ulong state = fd_failover_channel_state( ctx->channel ); + ulong generation = fd_failover_channel_generation( ctx->channel ); + if( FD_LIKELY( state==ctx->session.state && generation==ctx->session.generation ) ) return; + ctx->session.state = state; + ctx->session.generation = generation; + ctx->request.sent = 0; + ctx->session.close_after_send = 0; + if( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_HANDOFF_RESULT ) ctx->tx.valid = 0; + if( FD_UNLIKELY( state==FD_FAILOVER_SESSION_PAIRED ) ) { + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + ctx->session.peer_boot_id = peer->boot_id; + ctx->session.peer_holds = !!peer->holds; + if( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_SENT ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_PENDING; + if( FD_UNLIKELY( ( ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY || + ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ) && + peer->holds ) ) ctx->promotion.holder_seen = 1; + if( FD_UNLIKELY( peer->holds && ( ctx->peer_history.valid ) ) ) + FD_LOG_NOTICE(( "peer boot %016lx authenticated running the identity, discarding saved final state from before its voting tenure, keeping coverage floors", peer->boot_id )); + if( FD_UNLIKELY( peer->holds ) ) { + ctx->peer_history.valid = 0; + } + } +} + +/* Returns the port a handoff dials, from `failover promote --port` if + given, else the default port. */ +static inline ushort +dial_port( fd_failover_tile_ctx_t const * ctx ) { + return ctx->cmd_port ? ctx->cmd_port : FD_FAILOVER_DEFAULT_PORT; +} + +static void +set_holds( fd_failover_tile_ctx_t * ctx, + int holds ) { + /* Publish whether the completed identity switch installed the target. */ + ctx->holds = holds; + ctx->hello.holds = (uchar)holds; + fd_failover_channel_set_holds( ctx->channel, holds ); +} + +/* Sets the replay slot past which the current step is overdue. */ +static void +deadline_start( fd_failover_tile_ctx_t * ctx, + ulong slots ) { + ctx->deadline_slot = ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) + ? FD_FAILOVER_SLOT_NULL + : fd_ulong_sat_add( ctx->replay_slot, slots ); +} + +/* Sets the slot deadline to slots past the current replay slot, unless + it is already set or no replay slot has been seen yet. */ +static void +set_step_deadline( fd_failover_tile_ctx_t * ctx, + ulong slots ) { + if( FD_LIKELY( ctx->deadline_slot!=FD_FAILOVER_SLOT_NULL ) ) return; + if( FD_UNLIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) ) return; + ctx->deadline_slot = fd_ulong_sat_add( ctx->replay_slot, slots ); +} + +static int +deadline_expired( fd_failover_tile_ctx_t const * ctx ) { + return ctx->deadline_slot!=FD_FAILOVER_SLOT_NULL && + ctx->replay_slot !=FD_FAILOVER_SLOT_NULL && + ctx->replay_slot>ctx->deadline_slot; +} + +/* Sets the promotion deadline in slots and on the monotonic clock. */ +static void +set_promote_deadline( fd_failover_tile_ctx_t * ctx, + long now ) { + set_step_deadline( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + if( FD_UNLIKELY( !ctx->promotion.until ) ) + ctx->promotion.until = fd_long_sat_add( now, (long)FD_FAILOVER_DEADLINE_SLOTS*FD_FAILOVER_DEADLINE_SLOT_NANOS ); +} + +/* Returns whether the promotion passed its slot deadline or its clock + deadline. */ +static int +promote_expired( fd_failover_tile_ctx_t const * ctx, + long now ) { + if( FD_LIKELY( nowpromotion.until ) ) return deadline_expired( ctx ); + if( FD_UNLIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) ) { + FD_LOG_WARNING(( "%s: the promotion ran out of time before replay started, wait until this machine has caught up", ctx->promotion.label )); + } else { + FD_LOG_WARNING(( "%s: the promotion ran out of time with replay at slot %lu, replay may be stuck, check that this machine is catching up", ctx->promotion.label, ctx->replay_slot )); + } + return 1; +} + +/* Marks the identity switch in flight as overdue, flags stuck and warns + once. We keep waiting for the admin tile's answer. */ +static void +switch_overdue( fd_failover_tile_ctx_t * ctx ) { + ctx->stuck = 1; + if( FD_LIKELY( ctx->id_switch.overdue ) ) return; + ctx->id_switch.overdue = 1; + char demotion[ 48 ]; + fd_cstr_printf( demotion, sizeof(demotion), NULL, "handoff %lu", ctx->handoff.id ); + FD_LOG_WARNING(( "%s: identity switch %lu to %s is overdue, its outcome is unknown, we keep waiting and refuse another switch, check the admin and sign tile logs", + ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? ctx->promotion.label : demotion, + ctx->id_switch.request_id, ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? "the target identity" : "the junk key" )); +} + +/* Only one control message can be outstanding at a time. Its recipient + is fixed even if the connection changes before it is sent. */ +static int +queue_control( fd_failover_tile_ctx_t * ctx, + ushort type, + fd_failover_peer_t const * to, + void const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( ctx->tx.valid || !payload_sz || payload_sz>sizeof(ctx->tx.payload) ) ) return -1; + ctx->tx.type = type; + ctx->tx.sz = (ushort)payload_sz; + ctx->tx.valid = 1; + ctx->tx.to = *to; + fd_memcpy( ctx->tx.payload, payload, payload_sz ); + return 0; +} + +/* Our DEMOTED goes to the handoff target. */ +static int +queue_demoted( fd_failover_tile_ctx_t * ctx ) { + return queue_control( ctx, (ushort)FD_FAILOVER_MSG_DEMOTED, &ctx->handoff.target, ctx->handoff.payload, ctx->handoff.payload_sz ); +} + +/* Tells the requester, whose member and boot are in to, that we recorded + its answer to our DEMOTED or that its handoff cannot go ahead. */ +static void +handoff_result_to( fd_failover_tile_ctx_t * ctx, + fd_failover_peer_t const * to, + ulong id, + ulong result ) { + fd_failover_handoff_result_t response = { .handoff_id=id, .result=result }; + if( FD_LIKELY( !queue_control( ctx, FD_FAILOVER_MSG_HANDOFF_RESULT, to, &response, sizeof(response) ) ) ) { + /* Keep the connection until the result has drained. */ + ctx->session.close_after_send = 1; + ctx->session.close_handoff_id = id; + } +} + +/* queue_reply queues our last response to a DEMOTED. If the control slot + is busy the response stays pending and step_controller tries again once + the slot drains. */ +static void +queue_reply( fd_failover_tile_ctx_t * ctx ) { + uchar payload[ sizeof(fd_failover_promote_rejected_t) ]; + ulong payload_sz = ctx->reply.type==(ushort)FD_FAILOVER_MSG_PROMOTE_ACK + ? fd_failover_promote_ack_encode( payload, ctx->reply.handoff_id ) + : fd_failover_promote_rejected_encode( payload, ctx->reply.handoff_id, ctx->reply.reason ); + ctx->reply.state = queue_control( ctx, ctx->reply.type, &ctx->reply.peer, payload, payload_sz ) ? FD_FAILOVER_REPLY_PENDING : FD_FAILOVER_REPLY_KEPT; +} + +/* Remembers our response to the DEMOTED with handoff_id from peer, + then queues it. */ +static void +finish_reply( fd_failover_tile_ctx_t * ctx, + fd_failover_peer_t const * peer, + ulong handoff_id, + ushort type, + uchar reason ) { + if( ctx->reply.state==FD_FAILOVER_REPLY_NONE || ctx->reply.peer.boot_id!=peer->boot_id || ctx->reply.handoff_id!=handoff_id || ctx->reply.type!=type ) + ctx->reply.log_at = 0L; + ctx->reply.peer = *peer; + ctx->reply.handoff_id = handoff_id; + ctx->reply.type = type; + ctx->reply.reason = reason; + queue_reply( ctx ); +} + +/* The name of a result the machine running the identity sent us for our + handoff request, or the one we sent, and what the operator can do about + it. */ +static char const * +control_result( ulong result, + char const ** hint ) { + switch( result ) { + case FD_ADMINCTL_RESULT_SUCCESS: + *hint = "the other machine recorded that we took the identity but this machine does not run it, check `failover status` on both machines"; + return "SUCCESS"; + case FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY: + *hint = "the other machine could not finish this handoff and may have given up the identity, check `failover status` there"; + return "PEER_UNREADY"; + case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: + *hint = "the other machine is busy with another transition, run `failover promote` here again once `failover status` there shows it idle"; + return "IN_PROGRESS"; + case FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER: + *hint = "the machine we dialed does not run the identity any more, check `failover status` on both machines"; + return "WRONG_HOLDER"; + case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: + *hint = "the other machine has no eligible final vote state to hand over and keeps the identity, its log says why"; + return "NO_FINAL_HISTORY"; + case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: + *hint = "this machine's replay is too far behind the other machine's last vote and it keeps the identity, run `failover promote` here again once this machine has caught up"; + return "REPLAY_BEHIND"; + default: + *hint = "the other machine answered with a result this machine does not know, run the same Firedancer version on both machines"; + return "UNKNOWN"; + } +} + +/* What a control message we just sent changes on our side. */ +static void +control_sent( fd_failover_tile_ctx_t * ctx, + long now ) { + switch( ctx->tx.type ) { + case (ushort)FD_FAILOVER_MSG_DEMOTED: + if( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_PENDING ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_SENT; + if( now>=ctx->handoff.log_at ) { + ctx->handoff.log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); + FD_LOG_NOTICE(( "handoff %lu: sent final state (DEMOTED) to peer boot %016lx, last vote slot %lu, we run the junk key and wait for its answer", + ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot )); + } + break; + case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK: + case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: + if( now>=ctx->reply.log_at ) { + ctx->reply.log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); + FD_LOG_NOTICE(( "handoff %lu: sent %s to peer boot %016lx, %s, waiting for its final confirmation", + ctx->reply.handoff_id, ctx->tx.type==FD_FAILOVER_MSG_PROMOTE_ACK ? "ACK" : "refusal", + ctx->reply.peer.boot_id, ctx->holds ? "we run the identity" : "we do not run the identity" )); + } + break; + case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: { + fd_failover_handoff_result_t result; + fd_memcpy( &result, ctx->tx.payload, sizeof(result) ); + if( now>=ctx->handoff.result_log_at ) { + ctx->handoff.result_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); + char const * hint; + FD_LOG_NOTICE(( "handoff %lu: sent %s confirmation%s%s%s to peer boot %016lx, closing connection after queued bytes drain", + result.handoff_id, result.result==FD_ADMINCTL_RESULT_SUCCESS ? "success" : "refusal", + result.result==FD_ADMINCTL_RESULT_SUCCESS ? "" : " (", + result.result==FD_ADMINCTL_RESULT_SUCCESS ? "" : control_result( result.result, &hint ), + result.result==FD_ADMINCTL_RESULT_SUCCESS ? "" : ")", ctx->tx.to.boot_id )); + } + break; + } + default: break; + } +} + +static void +pending_flush( fd_failover_tile_ctx_t * ctx, + long now ) { + if( FD_LIKELY( !ctx->tx.valid ) ) return; + if( FD_UNLIKELY( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED || + fd_failover_channel_tx_pending( ctx->channel ) ) ) return; + /* We send a queued message only to the member and boot it was queued for. */ + if( FD_UNLIKELY( !peer_eq( &ctx->tx.to, ctx->session.peer_boot_id, fd_failover_channel_peer_hello( ctx->channel )->junk_pubkey ) ) ) return; + if( FD_UNLIKELY( fd_failover_channel_send( ctx->channel, now, ctx->tx.type, ctx->tx.payload, ctx->tx.sz ) ) ) return; + control_sent( ctx, now ); + ctx->tx.valid = 0; +} + +static void +prepare_history( fd_failover_tile_ctx_t * ctx, + fd_votor_hist_msg_t const * hist ) { + uchar state[ FD_FAILOVER_ALPENGLOW_STATE_MAX ]; + ulong state_sz = 0UL; + if( FD_UNLIKELY( ag_hist_ser( &hist->hist, state, FD_FAILOVER_ALPENGLOW_STATE_MAX, &state_sz ) || + ag_hist_tip( &hist->hist )!=hist->vote_slot ) ) { + /* Warn once until a good frame arrives. */ + if( FD_UNLIKELY( !ctx->hist_warned ) ) FD_LOG_WARNING(( "votor produced a vote history that does not match its slot metadata, until a good one arrives a handoff hands over nothing" )); + ctx->hist_warned = 1; + return; + } + ctx->hist_warned = 0; + fd_memcpy( ctx->current_history.state, state, state_sz ); + ctx->current_history.valid = 1; + ctx->current_history.tip = hist->vote_slot; + ctx->current_history.sz = state_sz; +} + +/* Takes in a votor_hist frame: replay and root slots, and the history + of the identity we hand off once the votor streams it in full. */ +static void +on_votor_hist_msg( fd_failover_tile_ctx_t * ctx, + fd_votor_hist_msg_t const * hist ) { + if( FD_LIKELY( hist->replay_slot!=FD_FAILOVER_SLOT_NULL && + ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || hist->replay_slot>ctx->replay_slot ) ) ) + ctx->replay_slot = hist->replay_slot; + if( FD_LIKELY( hist->root_slot!=FD_FAILOVER_SLOT_NULL ) ) ctx->root_slot = hist->root_slot; + /* Histories of another key can arrive during preparation or after the + switch. */ + if( FD_UNLIKELY( !fd_memeq( hist->identity.uc, ctx->hello.staked_pubkey, 32UL ) ) ) return; + int producing = ctx->holds || ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH; + if( FD_LIKELY( hist->has_vote && hist->vote_slot!=FD_FAILOVER_SLOT_NULL ) ) { + /* Every vote here is one we signed, so a later promotion has to cover it. */ + ctx->last_vote_slot = hist->vote_slot; + if( FD_LIKELY( ctx->own_floor==FD_FAILOVER_SLOT_NULL || hist->vote_slot>ctx->own_floor ) ) ctx->own_floor = hist->vote_slot; + prepare_history( ctx, hist ); + } else if( FD_UNLIKELY( producing && hist->vote_slot!=FD_FAILOVER_SLOT_NULL && ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ) ) { + /* The first history with a vote in it while we run the identity, + cast before we tracked our votes. */ + ctx->last_vote_slot = hist->vote_slot; + if( FD_LIKELY( ctx->own_floor==FD_FAILOVER_SLOT_NULL || hist->vote_slot>ctx->own_floor ) ) ctx->own_floor = hist->vote_slot; + prepare_history( ctx, hist ); + } else if( FD_UNLIKELY( producing && ctx->current_history.valid && hist->vote_slot!=FD_FAILOVER_SLOT_NULL && hist->vote_slot>=ctx->last_vote_slot ) ) { + /* A frame with no new vote can still change the history, after a LEADER or + a vote marked unsent. */ + ctx->last_vote_slot = hist->vote_slot; + prepare_history( ctx, hist ); + } +} + +/* Sends history or a forced empty-history bound to votor, for the target + identity. The response carries the request ID to distinguish late + replies. Returns that ID, or ULONG_MAX if there was no request to + send. */ +static ulong +publish_adopt_state( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + if( FD_UNLIKELY( ctx->adopt_out_idx==ULONG_MAX || + !ctx->promotion.adopt.sz ) ) return ULONG_MAX; + + if( FD_UNLIKELY( !++ctx->adopt_request_id ) ) ctx->adopt_request_id++; + uchar * req = fd_chunk_to_laddr( ctx->adopt_out_mem, ctx->adopt_out_chunk ); + ulong sz = FD_VOTOR_ADOPT_REQ_HDR_SZ+ctx->promotion.adopt.sz; + fd_memcpy( req, ctx->hello.staked_pubkey, FD_VOTOR_ADOPT_REQ_HDR_SZ ); + fd_memcpy( req+FD_VOTOR_ADOPT_REQ_HDR_SZ, ctx->promotion.adopt.state, ctx->promotion.adopt.sz ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->adopt_out_idx, ctx->adopt_request_id, ctx->adopt_out_chunk, sz, 0UL, tspub, tspub ); + ctx->adopt_out_chunk = fd_dcache_compact_next( ctx->adopt_out_chunk, sz, ctx->adopt_out_chunk0, ctx->adopt_out_wmark ); + ctx->adopt_result_fresh = 0; + return ctx->adopt_request_id; +} + +/* Asks the admin tile to switch to the given key. Only the public key + goes out, the signing tile preloaded both keypairs. Only one request + can be in flight, and the reply comes back with the same id so a late + reply can be told apart. */ +static ulong +request_switch( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem, + ulong key ) { + if( FD_UNLIKELY( ctx->admin_out_idx==ULONG_MAX || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) return ULONG_MAX; + + if( FD_UNLIKELY( !++ctx->id_switch.request_id ) ) ctx->id_switch.request_id++; + fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->admin_out_mem, ctx->admin_out_chunk ); + fd_memset( out, 0, sizeof(*out) ); + out->nonce = ctx->id_switch.request_id; + fd_failover_switch_req_t req; + fd_memcpy( req.identity, key==FD_FAILOVER_SWITCH_KEY_STAKED ? ctx->hello.staked_pubkey : ctx->hello.junk_pubkey, 32UL ); + req.epoch = ctx->id_switch.epoch; + fd_memcpy( out->payload, &req, sizeof(req) ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_SWITCH_REQ, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub ); + ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark ); + /* Keep the switch pending until admin responds to this request ID. */ + ctx->id_switch.pending_key = key; + ctx->id_switch.fresh = 0; + return ctx->id_switch.request_id; +} + +/* Ends the switch in flight when nonce answers it and returns 1 if the + caller should take the result. We drop a reply to another request, + and the reply to a switch that set-identity overtook. */ +static int +on_switch_resp_msg( fd_failover_tile_ctx_t * ctx, + ulong nonce ) { + if( FD_UNLIKELY( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT || + nonce!=ctx->id_switch.request_id ) ) { + FD_LOG_WARNING(( "dropping a stale identity switch answer" )); + return 0; + } + /* A matching response ends the pending switch. */ + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT; + if( FD_UNLIKELY( ctx->id_switch.discard ) ) { + ctx->id_switch.discard = 0; + return 0; + } + ctx->id_switch.fresh = 1; + return 1; +} + +/* set-identity installed a key, or failover promote picked a + target, so we start over as if this machine had rebooted. We take a + new boot id, so the other machine treats anything bound to the old one + as from a restarted peer, drop everything in flight, and take the + target that admin left. A peer with another target fails HELLO. */ +static void +on_operator_action_msg( fd_failover_tile_ctx_t * ctx, + fd_failover_operator_t const * operator ) { + if( FD_UNLIKELY( operator->epoch<=ctx->id_switch.epoch ) ) return; + ctx->id_switch.epoch = operator->epoch; + ctx->id_switch.discard = ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT; + ctx->id_switch.fresh = 0; + ctx->id_switch.overdue = 0; + + long now = fd_failover_clock(); + fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); + fd_failover_channel_hangup( ctx->channel, now ); + + /* Whatever was in flight with the other machine is dropped here, but + the other machine may already have acted on it. */ + int promoting = ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY || + ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT || + ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH; + int requesting = !!ctx->request.id; + int handing = ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE; + int replying = ctx->reply.state==FD_FAILOVER_REPLY_PENDING; + if( FD_UNLIKELY( requesting || handing || promoting || replying ) ) { + char request[ 64 ], handoff[ 64 ], reply[ 64 ]; /* a 20 digit id fits */ + fd_cstr_printf( request, sizeof(request), NULL, " our handoff request %lu,", ctx->request.id ); + fd_cstr_printf( handoff, sizeof(handoff), NULL, " our handoff %lu awaiting its answer,", ctx->handoff.id ); + fd_cstr_printf( reply, sizeof(reply), NULL, " our unsent reply to handoff %lu,", ctx->reply.handoff_id ); + FD_LOG_WARNING(( "the identity changed while failover was busy, dropping%s%s%s%s%s%s the other machine may already run the identity, " + "check `failover status` on both machines", + requesting ? request : "", handing ? handoff : "", + promoting ? " the " : "", promoting ? ctx->promotion.label : "", promoting ? " in progress," : "", + replying ? reply : "" )); + } + + fd_memset( &ctx->request, 0, sizeof(ctx->request) ); + fd_memset( &ctx->handoff, 0, sizeof(ctx->handoff) ); + fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) ); + fd_memset( &ctx->reply, 0, sizeof(ctx->reply) ); + fd_memset( &ctx->tx, 0, sizeof(ctx->tx) ); + ctx->peer_history.valid = 0; + ctx->current_history.valid = 0; + ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; + ctx->taken = 0; + ctx->last_requested = 0; + ctx->cmd_addr = 0U; + ctx->cmd_port = (ushort)0; + ctx->session.close_after_send = 0; + ctx->adopt_result_fresh = 0; + ctx->action = FD_FAILOVER_ACTION_IDLE; + ctx->stuck = 0; + ctx->deadline_slot = FD_FAILOVER_SLOT_NULL; + /* What gossip told us is about the old failover identity. */ + if( FD_UNLIKELY( !fd_memeq( operator->failover_identity, ctx->hello.staked_pubkey, 32UL ) ) ) { + ctx->staked_addr = 0U; + ctx->staked_seen_at = 0L; + ctx->peer_floor = FD_FAILOVER_SLOT_NULL; + ctx->own_floor = FD_FAILOVER_SLOT_NULL; + } + + ulong boot_id = ctx->hello.boot_id; + do FD_TEST( fd_rng_secure( &ctx->hello.boot_id, sizeof(ulong) ) ); while( FD_UNLIKELY( !ctx->hello.boot_id || ctx->hello.boot_id==boot_id ) ); + fd_memcpy( ctx->hello.staked_pubkey, operator->failover_identity, 32UL ); + fd_memcpy( ctx->installed, operator->identity, 32UL ); + ctx->holds = fd_memeq( operator->identity, operator->failover_identity, 32UL ); + ctx->hello.holds = (uchar)ctx->holds; + fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 }; + if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, (fd_failover_proof_signer_t){ .ctx=ctx, .sign_fn=sign_proof }, &ctx->hello ) ) ) { + FD_LOG_ERR(( "failover TLS setup failed after the identity changed" )); + } + sync_session( ctx ); + + FD_BASE58_ENCODE_32_BYTES( operator->identity, identity ); + FD_BASE58_ENCODE_32_BYTES( operator->failover_identity, failover_identity ); + FD_LOG_NOTICE(( "failover now hands off `%s` and this machine runs `%s`, with a new boot %016lx, " + "anything in progress was dropped and the other machine sees a restart", + failover_identity, identity, ctx->hello.boot_id )); +} + +/* Our DEMOTED got its response or cannot get one any more. */ +static void +handoff_resolved( fd_failover_tile_ctx_t * ctx, + ulong result ) { + if( FD_UNLIKELY( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ) ) ctx->tx.valid = 0; + /* The handoff outcome ends the ACK wait, this machine keeps its junk key. */ + if( FD_LIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) ctx->action = FD_FAILOVER_ACTION_IDLE; + ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE; + ctx->handoff.result = result; +} + +/* A demotion that ends without sending final state. Nobody can + promote on it, and the machine that asked is told at once rather than + at its deadline. */ +static void +demotion_abort( fd_failover_tile_ctx_t * ctx ) { + if( FD_UNLIKELY( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) ) { + handoff_result_to( ctx, &ctx->handoff.target, ctx->handoff.id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + } + /* End the failed demotion without changing the installed identity. */ + ctx->action = FD_FAILOVER_ACTION_IDLE; + ctx->stuck = 1; + ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE; +} + +/* Each demotion step gets a full deadline, in slots and on our clock. */ +static void +demotion_deadline_start( fd_failover_tile_ctx_t * ctx ) { + deadline_start( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + ctx->handoff.until = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_CHANNEL_IDLE_NANOS ); +} + +/* Give up the identity for the peer that asked, on the session it asked + on. The switch itself is asked for from step_controller, + DEMOTED goes to the peer boot we see now. */ +static void +start_demotion( fd_failover_tile_ctx_t * ctx, + ulong handoff_id ) { + fd_memset( &ctx->handoff, 0, sizeof(ctx->handoff) ); + ctx->handoff.id = handoff_id; + ctx->handoff.result = FD_FAILOVER_HANDOFF_NONE; /* until its DEMOTED goes out */ + ctx->handoff.delivery = FD_FAILOVER_DELIVERY_PENDING; + ctx->handoff.target.boot_id = ctx->session.peer_boot_id; + fd_memcpy( ctx->handoff.target.junk, fd_failover_channel_peer_hello( ctx->channel )->junk_pubkey, 32UL ); + ctx->last_requested = 0; + ctx->stuck = 0; + demotion_deadline_start( ctx ); + /* Start the junk key switch before sending any final state. */ + ctx->action = FD_FAILOVER_ACTION_DEMOTE_SWITCH; +} + +/* Returns the slot the other machine's replay has to reach before it + adopts what we hand over. */ +static ulong +ready_slot( fd_failover_tile_ctx_t const * ctx ) { + ag_hist_t hist[1]; + if( FD_UNLIKELY( ag_hist_de( ctx->current_history.state, ctx->current_history.sz, hist ) ) ) return ctx->last_vote_slot; + return hist->anchor; +} + +/* The final history has to be the history of our last vote. */ +static int +final_history_ok( fd_failover_tile_ctx_t const * ctx ) { + return ctx->current_history.valid && ctx->current_history.sz && + ctx->last_vote_slot!=FD_FAILOVER_SLOT_NULL && + ctx->current_history.tip==ctx->last_vote_slot; +} + +/* Called once the junk key is installed everywhere. Our latest history is + the final one, we hand it to the peer. */ +static void +demotion_switched( fd_failover_tile_ctx_t * ctx ) { + /* The junk key is installed and votor has drained. */ + set_holds( ctx, 0 ); + FD_LOG_NOTICE(( "handoff %lu: the vote history stream drained, this machine runs its junk key now", ctx->handoff.id )); + + int final_ok = final_history_ok( ctx ); + + ctx->handoff.payload_sz = !final_ok ? 0UL : + fd_failover_demoted_encode( ctx->handoff.payload, ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot, + ctx->current_history.state, ctx->current_history.sz ); + if( FD_UNLIKELY( !ctx->handoff.payload_sz ) ) { + /* We have no final history to hand over, or the one we have is not the + history of our last vote. The peer gets no final state and cannot + promote. */ + FD_LOG_WARNING(( "handoff %lu has no final vote history for its last vote %lu, sending nothing, nobody votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both", ctx->handoff.id, ctx->last_vote_slot )); + /* End demotion without sending final state. */ + demotion_abort( ctx ); + return; + } + + ctx->handoff.result = FD_FAILOVER_HANDOFF_PENDING; + /* The control slot may still be holding an unsent response, so if this + fails we retry from the wait. */ + (void)queue_demoted( ctx ); + /* Final state is ready, wait for the bound peer to accept or refuse it. */ + ctx->action = FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK; + demotion_deadline_start( ctx ); +} + +/* Returns the name of a refusal reason for the log. */ +static char const * +reject_name( uint reason ) { + static char const * const names[ FD_FAILOVER_REJECT_CNT ] = { + "none", "busy", "holds identity", "replay behind", + "adoption mismatch", "adoption failed", "switch failed" + }; + return reasonrequest.until ) ) fd_failover_channel_init_dialer( ctx->channel, ctx->request.addr, dial_port( ctx ) ); + ctx->action = FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT; + ctx->request.until = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_CHANNEL_IDLE_NANOS ); +} + +/* Give up the promotion. If the peer's DEMOTED asked for it, we tell it + why and drop the history it handed us, because after a refusal + either machine may recover on its own without reporting its later + votes. */ +static void +reject_promotion( fd_failover_tile_ctx_t * ctx, + uchar reason, + int bad_history ) { + if( FD_UNLIKELY( bad_history && ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ) ctx->peer_history.valid = 0; + ctx->stuck = 1; + /* A failed promotion returns to idle without the identity. */ + ctx->action = FD_FAILOVER_ACTION_IDLE; + if( FD_UNLIKELY( !ctx->promotion.from_peer ) ) { + FD_LOG_WARNING(( "%s: promotion failed (%s), this machine does not take the identity, check that one machine votes and see `failover status`, " + "run `failover promote --force` again once this machine has caught up and the other cannot sign", ctx->promotion.label, reject_name( reason ) )); + return; + } + ctx->peer_history.valid = 0; + ctx->promotion.from_peer = 0; + /* A failed handoff waits for the other machine to confirm our refusal. */ + request_wait_result( ctx ); + if( FD_UNLIKELY( reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { + FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), this machine does not take the identity, another machine may run it, " + "check `failover status` on both machines", ctx->promotion.handoff_id, reject_name( reason ) )); + } else { + FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), this machine does not take the identity and the peer already gave it up, " + "nothing votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both, check `failover status` on both machines first", + ctx->promotion.handoff_id, reject_name( reason ) )); + } + finish_reply( ctx, &ctx->promotion.peer, ctx->promotion.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason ); +} + +/* The coverage floor is the higher of the peer's reported last vote and our highest signed vote. */ +static ulong +coverage_floor( fd_failover_tile_ctx_t const * ctx ) { + if( FD_UNLIKELY( ctx->peer_floor==FD_FAILOVER_SLOT_NULL ) ) return ctx->own_floor; + if( FD_UNLIKELY( ctx->own_floor ==FD_FAILOVER_SLOT_NULL ) ) return ctx->peer_floor; + return fd_ulong_max( ctx->peer_floor, ctx->own_floor ); +} + +/* Records the finality anchor of the saved history we adopt, which replay + has to reach first. */ +static void +promote_start( fd_failover_tile_ctx_t * ctx ) { + ctx->adopt_anchor = FD_FAILOVER_SLOT_NULL; + ctx->empty_vote_after = FD_FAILOVER_SLOT_NULL; + if( FD_LIKELY( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) { + ag_hist_t hist[1]; + if( FD_LIKELY( !ag_hist_de( ctx->promotion.adopt.state, ctx->promotion.adopt.sz, hist ) ) ) ctx->adopt_anchor = hist->anchor; + } +} + +static int +replay_ready( fd_failover_tile_ctx_t * ctx ) { + if( FD_LIKELY( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) + return ctx->adopt_anchor==FD_FAILOVER_SLOT_NULL || ctx->replay_slot>=ctx->adopt_anchor; + ulong floor = ctx->promotion.floor; + ctx->empty_vote_after = floor==FD_FAILOVER_SLOT_NULL ? ctx->replay_slot : fd_ulong_max( ctx->replay_slot, floor ); + if( FD_UNLIKELY( floor==FD_FAILOVER_SLOT_NULL ) ) { + FD_LOG_WARNING(( "%s: --force accepts empty Alpenglow history, no vote or leader window at or below replay slot %lu, no earlier vote as the identity " + "is known, so a vote the peer cast above that slot is not covered, local vote marks and stronger local bounds remain", + ctx->promotion.label, ctx->empty_vote_after )); + } else { + FD_LOG_WARNING(( "%s: --force accepts empty Alpenglow history, no vote or leader window at or below slot %lu, which covers replay at slot %lu and the " + "last vote known as the identity at slot %lu, local vote marks and stronger local bounds remain", + ctx->promotion.label, ctx->empty_vote_after, ctx->replay_slot, floor )); + } + FD_STORE( ulong, ctx->promotion.adopt.state, ctx->empty_vote_after ); + ctx->promotion.adopt.sz = FD_VOTOR_ADOPT_EMPTY_SZ; + return 1; +} + +/* Logs what the votor adopted. */ +static void +log_adopted( fd_failover_tile_ctx_t const * ctx ) { + ulong bound = ctx->adopt_result.vote_bound; + if( FD_UNLIKELY( ctx->promotion.source==FD_FAILOVER_SOURCE_EMPTY ) ) { + FD_LOG_NOTICE(( "%s: the votor started an empty vote history, no vote at or below slot %lu", ctx->promotion.label, bound )); + } else if( FD_LIKELY( bound==FD_FAILOVER_SLOT_NULL ) ) { + FD_LOG_NOTICE(( "%s: the votor adopted the vote history ending at slot %lu, finality anchor at slot %lu", ctx->promotion.label, ctx->adopt_result.vote_slot, ctx->adopt_anchor )); + } else { + FD_LOG_NOTICE(( "%s: the votor adopted the vote history ending at slot %lu, finality anchor at slot %lu, no vote at or below slot %lu", ctx->promotion.label, + ctx->adopt_result.vote_slot, ctx->adopt_anchor, bound )); + } +} + +/* Take the identity with the history from source. A peer is given + when that member's DEMOTED with handoff_id asked for it, it gets our + response. An operator promotion passes NULL and 0. force is promote + --force. */ +static void +start_promotion( fd_failover_tile_ctx_t * ctx, + ulong source, + fd_failover_peer_t const * peer, + ulong handoff_id, + int force ) { + int from_peer = !!peer; + fd_failover_peer_t bound = {0}; + if( from_peer ) bound = *peer; + fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) ); + if( source==FD_FAILOVER_SOURCE_PEER ) ctx->promotion.adopt = ctx->peer_history; + else ctx->promotion.adopt.tip = FD_FAILOVER_SLOT_NULL; + if( from_peer ) { + ctx->promotion.peer = bound; + ctx->promotion.handoff_id = handoff_id; + fd_cstr_printf( ctx->promotion.label, sizeof(ctx->promotion.label), NULL, "handoff %lu", handoff_id ); + } else { + fd_cstr_printf( ctx->promotion.label, sizeof(ctx->promotion.label), NULL, "operator promotion" ); + } + ctx->promotion.source = source; + ctx->promotion.from_peer = from_peer; + ctx->promotion.force = force; + ctx->promotion.floor = coverage_floor( ctx ); + /* Until this tenure votes, its final history is exactly the adopted one, + not a history cached from an earlier tenure. */ + ctx->current_history = ctx->promotion.adopt; + ctx->last_vote_slot = ctx->promotion.adopt.tip; + ctx->stuck = 0; + promote_start( ctx ); + deadline_start( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + /* Replay must reach the selected history before adoption. */ + ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY; + /* A vote history waits for replay to reach its finality anchor. */ + if( FD_UNLIKELY( from_peer && ctx->adopt_anchor!=FD_FAILOVER_SLOT_NULL && + ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || ctx->replay_slotadopt_anchor ) ) ) { + FD_LOG_NOTICE(( "%s: the promotion waits for replay to reach the finality anchor at slot %lu before it adopts %s", ctx->promotion.label, ctx->adopt_anchor, source_name( source ) )); + } +} + +/* Forced promotion tries saved peer history, then bounded empty history. + Enter only before adoption is sent or after its response. A planned + handoff adopts its bound final state or refuses. */ +static int +promote_fallback( fd_failover_tile_ctx_t * ctx, + char const * reason ) { + if( FD_UNLIKELY( ctx->promotion.from_peer || !ctx->promotion.force ) ) return 0; + if( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) { + FD_LOG_NOTICE(( "%s: %s is not eligible (%s), trying %s", ctx->promotion.label, + source_name( ctx->promotion.source ), reason, source_name( FD_FAILOVER_SOURCE_EMPTY ) )); + if( ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ctx->peer_history.valid = 0; + /* Forced recovery falls back to a bound covering known votes. */ + ctx->promotion.source = FD_FAILOVER_SOURCE_EMPTY; + } else { + return 0; + } + ctx->promotion.adopt.valid = 0; + ctx->promotion.adopt.tip = FD_FAILOVER_SLOT_NULL; + ctx->promotion.adopt.sz = 0UL; + ctx->current_history = ctx->promotion.adopt; + ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; + ctx->promotion.retry = 0; + ctx->adopt_result_fresh = 0; + promote_start( ctx ); + /* Retry adoption from the fallback source after the prior attempt has ended. */ + ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY; + return 1; +} + +/* Returns whether adoption covers the highest vote the peer reported or + we signed. Forced empty history covers it through its vote bound. */ +static int +floor_covered( fd_failover_tile_ctx_t const * ctx ) { + ulong floor = ctx->promotion.floor; + ulong tip = ctx->promotion.source==FD_FAILOVER_SOURCE_EMPTY ? ctx->adopt_result.vote_bound + : ctx->adopt_result.vote_slot; + return floor==FD_FAILOVER_SLOT_NULL || ( tip!=FD_FAILOVER_SLOT_NULL && tip>=floor ); +} + +/* Whether a peer running the identity authenticated after this + promotion started, which stops it unless it is forced. */ +static int +promote_holder_seen( fd_failover_tile_ctx_t const * ctx ) { + if( FD_UNLIKELY( ctx->promotion.force ) ) return 0; + if( FD_UNLIKELY( ctx->promotion.holder_seen ) ) return 1; + return 0; +} + +static void +step_demote_drain( fd_failover_tile_ctx_t * ctx ) { + set_step_deadline( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + int expired = deadline_expired( ctx ) || ( ctx->handoff.until && fd_failover_clock()>=ctx->handoff.until ); + /* The watermark is the votor's output sequence when it stopped + signing. */ + if( FD_UNLIKELY( !fd_seq_ge( fd_seq_inc( ctx->hist_seen_seq, 1UL ), ctx->id_switch.result.history_watermark ) ) ) { + if( FD_UNLIKELY( expired ) ) { + FD_LOG_WARNING(( "handoff %lu: the vote history stream did not report its last votes in time, up to seq %lu, sending nothing, nobody votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both", ctx->handoff.id, ctx->id_switch.result.history_watermark )); + ctx->id_switch.fresh = 0; + ctx->id_switch.overdue = 0; + set_holds( ctx, 0 ); + demotion_abort( ctx ); + } + return; + } + ctx->id_switch.fresh = 0; + ctx->id_switch.overdue = 0; + demotion_switched( ctx ); +} + +static void +step_demote_switch( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + set_step_deadline( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + int expired = deadline_expired( ctx ) || ( ctx->handoff.until && fd_failover_clock()>=ctx->handoff.until ); + if( FD_UNLIKELY( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.fresh ) ) { + if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_JUNK )==ULONG_MAX ) ) { + /* The switch request did not go out and we still have the + identity, so we send no final state. */ + FD_LOG_WARNING(( "an identity switch could not be requested, this machine keeps the identity" )); + /* End demotion without sending final state. */ + demotion_abort( ctx ); + return; + } + FD_LOG_NOTICE(( "handoff %lu: asked the admin tile to install the junk identity", ctx->handoff.id )); + return; + } + if( FD_LIKELY( !ctx->id_switch.fresh ) ) { + /* No response yet, keep waiting. Past the deadline we raise stuck. */ + if( FD_UNLIKELY( expired ) ) switch_overdue( ctx ); + return; + } + if( FD_UNLIKELY( ctx->id_switch.result.result!=FD_FAILOVER_SWITCH_OK ) ) { + /* The switch failed and we still have the identity, so we send no + final state. */ + ctx->id_switch.fresh = 0; + ctx->id_switch.overdue = 0; + FD_LOG_WARNING(( "the identity switch for handoff %lu failed (%s), this machine keeps the identity", ctx->handoff.id, + ctx->id_switch.result.result==FD_FAILOVER_SWITCH_ERR_DISABLED ? "failover disabled" : + ctx->id_switch.result.result==FD_FAILOVER_SWITCH_ERR_AUTHORIZED_VOTER ? "the identity is an authorized voter" : "unknown" )); + /* End demotion without sending final state. */ + demotion_abort( ctx ); + return; + } + /* The junk key is installed everywhere. */ + fd_memcpy( ctx->installed, ctx->hello.junk_pubkey, 32UL ); + FD_LOG_NOTICE(( "handoff %lu: the junk identity is installed, waiting for the vote history stream to drain up to seq %lu", ctx->handoff.id, ctx->id_switch.result.history_watermark )); + ctx->action = FD_FAILOVER_ACTION_DEMOTE_DRAIN; + demotion_deadline_start( ctx ); + step_demote_drain( ctx ); +} + +static void +step_demote_wait_ack( fd_failover_tile_ctx_t * ctx ) { + /* If the peer goes quiet we do not undo the demotion, we only raise + the alarm. The identity is already gone from this machine. */ + set_step_deadline( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + /* A new peer boot cannot respond to this DEMOTED. Its earlier boot may + already have taken the identity and voted before losing its ACK. + The junk key is generated at every boot, so a paired peer with + another boot id is the member's new boot: pairing proved it holds + the same staked key. */ + if( FD_UNLIKELY( fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED && + ctx->session.peer_boot_id!=ctx->handoff.target.boot_id ) ) { + FD_LOG_WARNING(( "the peer restarted before it answered handoff %lu and may have voted before it restarted, neither machine runs the identity now, " + "nothing votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both", ctx->handoff.id )); + ctx->peer_history.valid = 0; + /* A new peer boot ends this ACK wait without proving whether it voted. */ + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_RESTARTED ); + return; + } + /* A reconnect from the same member can finish its handoff. */ + if( FD_UNLIKELY( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_PENDING && !ctx->tx.valid ) ) { + (void)queue_demoted( ctx ); + } + if( FD_UNLIKELY( deadline_expired( ctx ) || ( ctx->handoff.until && fd_failover_clock()>=ctx->handoff.until ) ) ) { + if( FD_UNLIKELY( !ctx->stuck ) ) + FD_LOG_WARNING(( "the peer has not answered handoff %lu, we run the junk key, the requester's request pauses at its deadline and " + "`failover promote` on the requester resumes it, `failover promote --force` only when its status shows `runs target identity: no` " + "and no promotion in progress, and only on the machine whose status shows a coverage floor, never both", ctx->handoff.id )); + ctx->stuck = 1; + } + return; +} + +static void +step_promote_wait_replay( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + long now = fd_failover_clock(); + set_promote_deadline( ctx, now ); + if( FD_UNLIKELY( promote_expired( ctx, now ) ) ) { + /* Replay timed out, fail the promotion without taking the identity. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_REPLAY_BEHIND, 0 ); + return; + } + /* Replay must start and reach the history's finality anchor. */ + if( FD_UNLIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) ) return; + if( FD_UNLIKELY( !replay_ready( ctx ) ) ) { + if( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) (void)promote_fallback( ctx, "replay has not reached its finality anchor" ); + return; + } + ulong id = publish_adopt_state( ctx, stem ); + if( FD_UNLIKELY( id==ULONG_MAX ) ) { + /* End the failed adoption attempt without taking the identity. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); + return; + } + FD_LOG_NOTICE(( "%s: asking the votor to adopt %s", ctx->promotion.label, source_name( ctx->promotion.source ) )); + ctx->promotion.adopt_id = id; + ctx->promotion.retry = 0; + /* The adoption request is published, wait for its response. */ + ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT; + return; +} + +static void +step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + long now = fd_failover_clock(); + set_promote_deadline( ctx, now ); + if( FD_UNLIKELY( promote_expired( ctx, now ) ) ) { + /* Adoption timed out, fail promotion without taking the identity. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); + return; + } + if( FD_UNLIKELY( ctx->promotion.retry ) ) { + /* Wait for replay before retrying a recoverable adoption error. */ + if( FD_LIKELY( ctx->replay_slot<=ctx->promotion.retry_slot ) ) return; + ulong id = publish_adopt_state( ctx, stem ); + if( FD_UNLIKELY( id==ULONG_MAX ) ) { + /* End the failed adoption attempt without taking the identity. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); + return; + } + /* Replay advanced, retry adoption without leaving the adoption wait. */ + ctx->promotion.adopt_id = id; + ctx->promotion.retry = 0; + return; + } + if( FD_LIKELY( !ctx->adopt_result_fresh ) ) return; /* Wait for an adoption response. */ + ctx->adopt_result_fresh = 0; + if( FD_UNLIKELY( ctx->adopt_result_id!=ctx->promotion.adopt_id ) ) return; /* Ignore responses to earlier attempts. */ + if( FD_UNLIKELY( ctx->adopt_result.result==FD_VOTOR_ADOPT_ERR_UNREPLAYED || + ctx->adopt_result.result==FD_VOTOR_ADOPT_ERR_CAPACITY ) ) { + /* Operator promotion tries a fallback before waiting for more replay. */ + if( promote_fallback( ctx, adopt_err_name( ctx->adopt_result.result ) ) ) return; + if( FD_UNLIKELY( !ctx->promotion.from_peer ) ) { + /* End the failed adoption attempt without taking the identity. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); + return; + } + if( FD_UNLIKELY( !ctx->promotion.retry_logged ) ) { + FD_LOG_NOTICE(( "%s: the votor is not ready to adopt the vote history (%s), asking again as replay moves", ctx->promotion.label, adopt_err_name( ctx->adopt_result.result ) )); + ctx->promotion.retry_logged = 1; + } + /* Keep the bound final state and wait for replay before retrying. */ + ctx->promotion.retry = 1; + ctx->promotion.retry_slot = ctx->replay_slot; + return; + } + if( FD_UNLIKELY( ctx->adopt_result.result!=FD_VOTOR_ADOPT_SUCCESS ) ) { + /* Try the next permitted source before failing promotion. */ + if( promote_fallback( ctx, adopt_err_name( ctx->adopt_result.result ) ) ) return; + FD_LOG_WARNING(( "%s: the votor refused %s (%s)", ctx->promotion.label, source_name( ctx->promotion.source ), adopt_err_name( ctx->adopt_result.result ) )); + ulong result = ctx->adopt_result.result; + /* Votor rejected the history, stop promotion and report any handoff refusal. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, + result==FD_VOTOR_ADOPT_ERR_DECODE || result==FD_VOTOR_ADOPT_ERR_INVALID || result==FD_VOTOR_ADOPT_ERR_STALE ); + return; + } + if( FD_UNLIKELY( ctx->promotion.adopt.tip!=FD_FAILOVER_SLOT_NULL && ctx->adopt_result.vote_slot!=ctx->promotion.adopt.tip ) ) { + /* Operator promotion may retry a different source after a tip mismatch. */ + if( promote_fallback( ctx, "adoption did not retain its final vote" ) ) return; + FD_LOG_WARNING(( "%s: the adopted vote history ends at slot %lu, the history says %lu, so it is a mismatch", ctx->promotion.label, ctx->adopt_result.vote_slot, ctx->promotion.adopt.tip )); + /* The adopted tip differs, fail promotion without installing the identity. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_MISMATCH, 1 ); + return; + } + log_adopted( ctx ); + int covered = floor_covered( ctx ); + if( FD_UNLIKELY( !covered && !ctx->promotion.from_peer && ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) { + if( promote_fallback( ctx, "its votes do not cover the known signing history" ) ) return; + } + if( FD_UNLIKELY( !covered && !ctx->promotion.force ) ) { + /* Only a handoff gets here. The tip of the peer's history does not + change, so we do not ask again and the deadline ends that wait. */ + if( FD_UNLIKELY( !ctx->promotion.floor_logged ) ) { + FD_LOG_WARNING(( "%s: %s ends at slot %lu, short of the coverage floor at slot %lu, the promotion stops at its deadline", ctx->promotion.label, + source_name( ctx->promotion.source ), ctx->adopt_result.vote_slot, ctx->promotion.floor )); + ctx->promotion.floor_logged = 1; + } + /* Stay in the adoption wait until the deadline expires. */ + ctx->promotion.retry = 1; + ctx->promotion.retry_slot = FD_FAILOVER_SLOT_NULL; + return; + } + if( FD_UNLIKELY( promote_holder_seen( ctx ) ) ) { + /* The identity may be held elsewhere now, so we do not install it + too. */ + FD_LOG_WARNING(( "%s: another machine may run the identity now, a peer running it authenticated during this promotion, " + "not installing it, check `failover status` on both machines", ctx->promotion.label )); + /* Another holder was seen, stop before installing the identity. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_HOLDS_IDENTITY, 0 ); + return; + } + if( FD_UNLIKELY( ctx->promotion.adopt.tip==FD_FAILOVER_SLOT_NULL ) ) ctx->last_vote_slot = ctx->adopt_result.vote_slot; + if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_STAKED )==ULONG_MAX ) ) { + /* The switch was not submitted, fail promotion without taking the identity. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); + return; + } + FD_LOG_NOTICE(( "%s: asking the admin tile to install the identity", ctx->promotion.label )); + /* Adoption passed, wait for admin to install the identity. */ + ctx->action = FD_FAILOVER_ACTION_PROMOTE_SWITCH; + return; +} + +static void +step_promote_switch( fd_failover_tile_ctx_t * ctx ) { + set_step_deadline( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + if( FD_LIKELY( !ctx->id_switch.fresh ) ) { + /* No response yet. Standing down here would tell the peer nobody + promoted while the identity may well be installed. So we keep + waiting and raise stuck past the deadline, the late response still + counts. */ + if( FD_UNLIKELY( deadline_expired( ctx ) || ( ctx->promotion.until && fd_failover_clock()>=ctx->promotion.until ) ) ) switch_overdue( ctx ); + return; + } + ctx->id_switch.fresh = 0; + ctx->id_switch.overdue = 0; + if( FD_UNLIKELY( ctx->id_switch.result.result!=FD_FAILOVER_SWITCH_OK ) ) { + /* Admin refused the switch, fail the promotion without the identity. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_SWITCH_FAILED, 0 ); + return; + } + /* A local recovery installs the identity outside a handoff. + Reauthenticate so a background session cannot retain our old HELLO. */ + if( !ctx->promotion.from_peer ) { + fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); + fd_failover_channel_hangup( ctx->channel, fd_failover_clock() ); + sync_session( ctx ); + } + /* Admin confirmed the identity is installed. */ + fd_memcpy( ctx->installed, ctx->hello.staked_pubkey, 32UL ); + set_holds( ctx, 1 ); + /* A new tenure starts, the peer history we kept is older than its votes. */ + ctx->peer_history.valid = 0; + ctx->taken = 0; + /* Local promotion is complete. */ + ctx->action = FD_FAILOVER_ACTION_IDLE; + ctx->stuck = 0; + if( FD_LIKELY( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) { + FD_LOG_NOTICE(( "%s: this machine runs the identity now, the adopted vote history ends at slot %lu", ctx->promotion.label, ctx->last_vote_slot )); + } else { + FD_LOG_NOTICE(( "%s: this machine runs the identity now with an empty history, no vote at or below slot %lu", ctx->promotion.label, ctx->empty_vote_after )); + } + if( FD_UNLIKELY( ctx->promotion.from_peer ) ) { + /* The ACK tells the other machine its handoff is done. We keep it as + our response, so the same DEMOTED again gets the ACK again. */ + ctx->promotion.from_peer = 0; + /* Keep the identity while the other machine confirms our ACK. */ + request_wait_result( ctx ); + finish_reply( ctx, &ctx->promotion.peer, ctx->promotion.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_ACK, (uchar)FD_FAILOVER_REJECT_NONE ); + } + return; +} + +/* Whether we can hand off now to a machine whose replay is at + replay_slot. Returns the refusal code or FD_ADMINCTL_RESULT_SUCCESS. */ +static ulong +handoff_check( fd_failover_tile_ctx_t * ctx, + ulong handoff_id, + ulong replay_slot, + long now ) { + ulong result = FD_ADMINCTL_RESULT_SUCCESS; + if( !final_history_ok( ctx ) ) result = FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY; + /* A machine this far behind could not reach the slot its promotion + waits for before the deadline. */ + else if( replay_slot==FD_FAILOVER_SLOT_NULL || + fd_ulong_sat_add( replay_slot, FD_FAILOVER_DEADLINE_SLOTS )=ctx->handoff_refuse_log_at ) { + ctx->handoff_refuse_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); + char tip[ 32 ], last[ 32 ]; + char const * why = ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ? "no vote is known to the failover controller, it cannot determine whether this machine has not voted or is unable to vote" : + !ctx->current_history.valid || !ctx->current_history.sz ? "a vote is known but its final state is unavailable" : + "the saved final state does not match the latest known vote"; + FD_LOG_WARNING(( "handoff %lu refused: %s (latest known vote %s, saved history tip %s), this machine keeps the identity, check local voting progress and vote-history logs before retrying", + handoff_id, why, slot_text( ctx->last_vote_slot, last ), + slot_text( ctx->current_history.valid ? ctx->current_history.tip : FD_FAILOVER_SLOT_NULL, tip ) )); + } else if( result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND && now>=ctx->handoff_refuse_log_at ) { + ctx->handoff_refuse_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); + char replay[ 32 ]; + FD_LOG_WARNING(( "handoff %lu refused: the other machine's replay is at slot %s, more than %lu slots behind our finality anchor at slot %lu, this machine keeps the identity, " + "run `failover promote` there again once it has caught up", + handoff_id, slot_text( replay_slot, replay ), FD_FAILOVER_DEADLINE_SLOTS, ready_slot( ctx ) )); + } + return result; +} + +/* Starts giving up the identity for the handoff a peer asked for on the + current session. */ +static void +accept_handoff( fd_failover_tile_ctx_t * ctx, + ulong handoff_id ) { + start_demotion( ctx, handoff_id ); + char member[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( ctx->handoff.target.junk, NULL, member ); + FD_LOG_NOTICE(( "handoff %lu: accepted request from member %s boot %016lx, preparing to give up the identity", ctx->handoff.id, member, ctx->handoff.target.boot_id )); +} + +/* Advances the current failover step. */ +static void +step_controller( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + if( FD_UNLIKELY( ctx->reply.state==FD_FAILOVER_REPLY_PENDING && !ctx->tx.valid ) ) queue_reply( ctx ); + if( FD_LIKELY( ctx->action==FD_FAILOVER_ACTION_IDLE ) ) return; + + switch( ctx->action ) { + + case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER: + case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: + return; /* Peer polling handles these waits. */ + + case FD_FAILOVER_ACTION_DEMOTE_SWITCH: + step_demote_switch( ctx, stem ); + return; + + case FD_FAILOVER_ACTION_DEMOTE_DRAIN: + step_demote_drain( ctx ); + return; + + case FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK: + step_demote_wait_ack( ctx ); + return; + + case FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY: + step_promote_wait_replay( ctx, stem ); + return; + + case FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT: + step_promote_wait_adopt( ctx, stem ); + return; + + case FD_FAILOVER_ACTION_PROMOTE_SWITCH: + step_promote_switch( ctx ); + return; + + default: FD_LOG_ERR(( "unexpected failover action %lu", ctx->action )); + } +} + +static void +request_end( fd_failover_tile_ctx_t * ctx, long now, ulong result ); + +/* Describes what our handoff request is waiting for, for logging. */ +static char const * +request_wait( fd_failover_tile_ctx_t const * ctx ) { + switch( ctx->action ) { + case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER: + if( !ctx->request.peer.boot_id ) return "no peer authenticated, no handoff request sent, we do not run the identity"; + if( !ctx->request.send_cnt ) return "peer authenticated, no handoff request sent, we do not run the identity"; + return "waiting for final state, the peer may still run the identity"; + case FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY: return "replay catch-up continues locally, we do not run the identity"; + case FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT: return "adoption continues locally, we do not run the identity"; + case FD_FAILOVER_ACTION_PROMOTE_SWITCH: return "the identity switch continues locally, its outcome is still unknown"; + case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: + return ctx->holds ? "we run the identity, waiting for the peer to confirm our ACK" + : "we do not run the identity, waiting for the peer to confirm our refusal"; + default: return "the local transition is retained"; + } +} + +/* What the operator can do about a paused request. */ +static char const * +request_next( fd_failover_tile_ctx_t const * ctx ) { + if( FD_UNLIKELY( ctx->request.peer_restarted ) ) { + if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) return "the authenticated peer changed, `failover promote` here ends the confirmation wait without changing our identity"; + return "the authenticated member or boot changed, `failover promote` here starts a new handoff"; + } + switch( ctx->action ) { + case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER: + if( !ctx->request.send_cnt ) return "nothing was sent and the other machine still runs the identity, `failover promote` here resumes the request, " + "and if that machine is down and cannot sign `failover promote --force` takes over"; + return "the other machine may already have given up the identity, check `failover status` there, `failover promote` here resumes the request"; + case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: + if( ctx->holds ) return "this machine runs the identity and votes, `failover promote` here asks the other machine again to confirm"; + if( ctx->reply.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) return "this machine refused the handoff because another machine may run the identity, " + "check `failover status` on both machines, `failover promote` here asks the other machine again to confirm"; + return "neither machine runs the identity, `failover promote` here asks the other machine again to confirm, " + "and `failover promote --force` on the one whose status shows a coverage floor, never both, takes the identity"; + default: + return "the local promotion goes on and dials again when it ends"; + } +} + +static void +request_pause( fd_failover_tile_ctx_t * ctx, + long now, + char const * why ) { + FD_LOG_WARNING(( "handoff %lu paused (%s): %s, dialing stopped, %s", ctx->request.id, why, request_wait( ctx ), request_next( ctx ) )); + /* Pause dialing without changing the action or the authenticated binding. */ + ctx->request.until = 0L; + ctx->stuck = 1; + fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); + fd_failover_channel_hangup( ctx->channel, now ); + sync_session( ctx ); +} + +/* handoff_result_to for the member and boot of the current session. */ +static void +handoff_result( fd_failover_tile_ctx_t * ctx, + ulong id, + ulong result ) { + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + fd_failover_peer_t to = { .boot_id=peer->boot_id }; + fd_memcpy( to.junk, peer->junk_pubkey, 32UL ); + handoff_result_to( ctx, &to, id, result ); +} + +/* An ACK or refusal counts only for the DEMOTED we sent, from the member + and boot it went to. */ +static int +demoted_answer_bound( fd_failover_tile_ctx_t const * ctx, + ulong handoff_id ) { + return ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE && handoff_id==ctx->handoff.id && + peer_eq( &ctx->handoff.target, ctx->session.peer_boot_id, fd_failover_channel_peer_hello( ctx->channel )->junk_pubkey ) && + ctx->action!=FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->action!=FD_FAILOVER_ACTION_DEMOTE_DRAIN; +} + +/* The peer's message handlers. Each returns 1 when the message was + handled, including one it ignored on purpose, and 0 when it was + malformed, which drops the session: these messages hand off the + identity so we do not guess at them. */ + +/* HANDOFF_REQUEST, the peer asks us to give up the identity. */ +static int +on_handoff_request_msg( fd_failover_tile_ctx_t * ctx, + ulong payload_sz, + long now ) { + fd_failover_handoff_request_t request; + if( FD_UNLIKELY( !fd_failover_handoff_request_decode( &request, ctx->rx, payload_sz ) ) ) return 0; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + if( FD_UNLIKELY( request.target_boot_id!=ctx->hello.boot_id ) ) { + handoff_result( ctx, request.handoff_id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + return 1; + } + if( FD_UNLIKELY( request.handoff_id==ctx->handoff.id && peer_eq( &ctx->handoff.target, peer->boot_id, peer->junk_pubkey ) ) ) { + /* A duplicate request keeps the same demotion and reuses its eventual response. */ + if( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH || ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN ) return 1; + if( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) { + if( !ctx->tx.valid ) (void)queue_demoted( ctx ); + } else { + handoff_result( ctx, request.handoff_id, ctx->handoff.result==FD_FAILOVER_HANDOFF_TAKEN + ? FD_ADMINCTL_RESULT_SUCCESS : FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + } + return 1; + } + ulong result = FD_ADMINCTL_RESULT_SUCCESS; + if( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT || ctx->tx.valid || ctx->session.close_after_send ) result = FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; + else if( !ctx->holds || peer->holds ) result = FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER; + /* Decline the request without changing the installed identity or action. */ + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + handoff_result( ctx, request.handoff_id, result ); + return 1; + } + result = handoff_check( ctx, request.handoff_id, request.replay_slot, now ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + handoff_result( ctx, request.handoff_id, result ); + return 1; + } + accept_handoff( ctx, request.handoff_id ); + return 1; +} + +/* HANDOFF_RESULT, the peer recorded how our request ended. */ +static int +on_handoff_result_msg( fd_failover_tile_ctx_t * ctx, + ulong payload_sz, + long now ) { + fd_failover_handoff_result_t result; + if( FD_UNLIKELY( !fd_failover_handoff_result_decode( &result, ctx->rx, payload_sz ) ) ) return 0; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + if( FD_UNLIKELY( !ctx->request.id || result.handoff_id!=ctx->request.id || + !peer_eq( &ctx->request.peer, peer->boot_id, peer->junk_pubkey ) ) ) return 1; + if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && + ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) return 1; + int failed = result.result!=FD_ADMINCTL_RESULT_SUCCESS || !ctx->holds; + if( FD_UNLIKELY( failed && ctx->holds ) ) { + FD_LOG_WARNING(( "handoff %lu: the other machine did not record that we took the identity and may run it again, check `failover status` on both machines at once", result.handoff_id )); + } else if( FD_UNLIKELY( failed && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && ctx->reply.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { + FD_LOG_WARNING(( "handoff %lu ended, this machine refused the final state (%s) because another machine may run the identity, and the other machine " + "recorded it, check `failover status` on both machines", result.handoff_id, reject_name( ctx->reply.reason ) )); + } else if( FD_UNLIKELY( failed && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) { + FD_LOG_WARNING(( "handoff %lu ended, this machine refused the final state (%s) and the other machine recorded it, neither machine runs the identity, " + "nothing votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both", result.handoff_id, reject_name( ctx->reply.reason ) )); + } else if( FD_UNLIKELY( failed && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY && + ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && ctx->request.send_cntrequest.send_cnt )); + return 1; + } else if( FD_UNLIKELY( failed ) ) { + char const * hint; + char const * name = control_result( result.result, &hint ); + FD_LOG_WARNING(( "handoff %lu ended (%s), %s", result.handoff_id, name, hint )); + } + /* A bound final result ends the request without changing our installed identity. */ + request_end( ctx, now, failed ? FD_FAILOVER_HANDOFF_DECLINED : FD_FAILOVER_HANDOFF_TAKEN ); + if( !failed ) + FD_LOG_NOTICE(( "handoff %lu complete: the other machine confirmed our ACK, we run the identity, connection closed", result.handoff_id )); + else + FD_LOG_NOTICE(( "handoff %lu: request ended, %s, connection closed", result.handoff_id, + ctx->holds ? "we still run the identity" : "we do not run the identity" )); + return 1; +} + +/* DEMOTED, the peer gave up the identity and hands us its final history. */ +static int +on_demoted_msg( fd_failover_tile_ctx_t * ctx, + ulong payload_sz ) { + /* The peer says it can no longer sign and hands us its history. */ + fd_failover_demoted_t demoted; + if( FD_UNLIKELY( !fd_failover_demoted_decode( &demoted, ctx->rx, payload_sz ) ) ) return 0; + /* HELLO pairs only members in the same mode, so a DEMOTED in the + other mode is malformed. */ + if( FD_UNLIKELY( demoted.mode!=(uchar)FD_FAILOVER_MODE_ALPENGLOW ) ) return 0; + ulong boot_id = ctx->session.peer_boot_id; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + /* Bind before deduplication. An unsolicited DEMOTED cannot change + this operation or replace its cached response. */ + if( FD_UNLIKELY( !ctx->request.id || ctx->request.id!=demoted.handoff_id || + !peer_eq( &ctx->request.peer, boot_id, peer->junk_pubkey ) ) ) return 1; + if( FD_UNLIKELY( demoted.target_boot_id!=ctx->hello.boot_id ) ) return 0; + /* The same DEMOTED again. Once we are done with it we repeat our + response. */ + if( FD_UNLIKELY( ctx->promotion.from_peer && ctx->promotion.peer.boot_id==boot_id && ctx->promotion.handoff_id==demoted.handoff_id ) ) return 1; + if( FD_UNLIKELY( ctx->reply.state!=FD_FAILOVER_REPLY_NONE && ctx->reply.handoff_id==demoted.handoff_id && peer_eq( &ctx->reply.peer, boot_id, peer->junk_pubkey ) ) ) { + queue_reply( ctx ); + return 1; + } + fd_failover_history_t history = { .valid=1, .tip=demoted.last_vote_slot, .sz=(ulong)demoted.state_len }; + fd_memcpy( history.state, ctx->rx+sizeof(fd_failover_demoted_t), history.sz ); + uchar reason = FD_FAILOVER_REJECT_NONE; + char const * why = NULL; + if( FD_UNLIKELY( ctx->holds ) ) { + reason = FD_FAILOVER_REJECT_HOLDS_IDENTITY; + why = "we run the identity"; + } else if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || + ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) { + reason = FD_FAILOVER_REJECT_BUSY; + why = "a transition or key switch is running here"; + } + if( FD_UNLIKELY( reason!=FD_FAILOVER_REJECT_NONE ) ) { + FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), %s, check `failover status` on both machines", demoted.handoff_id, reject_name( reason ), why )); + finish_reply( ctx, &ctx->request.peer, demoted.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason ); + return 1; + } + FD_LOG_NOTICE(( "handoff %lu: accepted the peer's final vote history ending at slot %lu, waiting for local adoption before taking the identity", demoted.handoff_id, history.tip )); + if( ctx->peer_floor==FD_FAILOVER_SLOT_NULL || history.tip>ctx->peer_floor ) ctx->peer_floor = history.tip; + ctx->peer_history = history; + ctx->session.peer_holds = 0; + /* The bound final state moves us from peer wait to replay wait. */ + start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &ctx->request.peer, demoted.handoff_id, 0 ); + return 1; +} + +/* PROMOTE_ACK, the peer took the identity we handed off. */ +static int +on_promote_ack_msg( fd_failover_tile_ctx_t * ctx, + ulong payload_sz ) { + /* The peer took the identity, so our DEMOTED did its job. An ACK + after the deadline still counts, and we ignore an ACK we are not + waiting for rather than drop the session. */ + fd_failover_promote_ack_t ack; + if( FD_UNLIKELY( !fd_failover_promote_ack_decode( &ack, ctx->rx, payload_sz ) ) ) return 0; + if( FD_UNLIKELY( !demoted_answer_bound( ctx, ack.handoff_id ) ) ) return 1; + FD_LOG_NOTICE(( "handoff %lu: received ACK, the peer took the identity, we keep the junk key", ack.handoff_id )); + /* The peer ACK ends the demotion wait and records that it took the identity. */ + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_TAKEN ); + ctx->taken = 1; + ctx->session.peer_holds = 1; + ctx->stuck = 0; + /* The peer votes past the final history we handed it from here on. */ + ctx->peer_history.valid = 0; + handoff_result( ctx, ack.handoff_id, FD_ADMINCTL_RESULT_SUCCESS ); + return 1; +} + +/* PROMOTE_REJECTED, the peer did not take the identity we handed off. */ +static int +on_promote_rejected_msg( fd_failover_tile_ctx_t * ctx, + ulong payload_sz ) { + /* The peer did not take the identity. Either machine may now recover + without telling the other, so we drop any history the peer handed us + earlier. */ + fd_failover_promote_rejected_t rej; + if( FD_UNLIKELY( !fd_failover_promote_rejected_decode( &rej, ctx->rx, payload_sz ) ) ) return 0; + if( FD_UNLIKELY( !demoted_answer_bound( ctx, rej.handoff_id ) ) ) return 1; + if( FD_UNLIKELY( rej.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { + FD_LOG_WARNING(( "the peer declined handoff %lu (%s), it saw another machine that may run the identity, this machine gave it up, " + "check `failover status` on both machines and the peer's log", rej.handoff_id, reject_name( rej.reason ) )); + } else { + FD_LOG_WARNING(( "the peer declined handoff %lu (%s), neither machine runs the identity now, nothing votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both, " + "see the peer's log first", rej.handoff_id, reject_name( rej.reason ) )); + } + ctx->peer_history.valid = 0; + /* The peer refusal ends the demotion wait, this machine keeps its junk key. */ + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_DECLINED ); + ctx->stuck = 1; + handoff_result( ctx, rej.handoff_id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + return 1; +} + +/* Handle a controller message. Anything malformed drops the session, + these messages hand off the identity so we do not guess at them. */ +static void +handle_control( fd_failover_tile_ctx_t * ctx, + ushort type, + ulong payload_sz, + long now ) { + int handled; + switch( type ) { + case (ushort)FD_FAILOVER_MSG_HANDOFF_REQUEST: handled = on_handoff_request_msg ( ctx, payload_sz, now ); break; + case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: handled = on_handoff_result_msg ( ctx, payload_sz, now ); break; + case (ushort)FD_FAILOVER_MSG_DEMOTED: handled = on_demoted_msg ( ctx, payload_sz ); break; + case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK: handled = on_promote_ack_msg ( ctx, payload_sz ); break; + case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: handled = on_promote_rejected_msg( ctx, payload_sz ); break; + default: handled = 0; break; + } + if( FD_LIKELY( handled ) ) return; + + /* Malformed control traffic drops the connection and retains local work. */ + fd_failover_channel_protocol_error( ctx->channel, now ); + sync_session( ctx ); +} + +/* End our handoff request with result, an FD_FAILOVER_HANDOFF_*, and + hang up. */ +static void +request_end( fd_failover_tile_ctx_t * ctx, + long now, + ulong result ) { + int failed = result!=FD_FAILOVER_HANDOFF_TAKEN; + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ); + ctx->request.result = result; + ctx->request.id = 0UL; + ctx->request.until = 0L; + ctx->tx.valid = 0; + ctx->session.close_after_send = 0; + if( ctx->reply.state==FD_FAILOVER_REPLY_PENDING ) ctx->reply.state = FD_FAILOVER_REPLY_KEPT; + /* Return to idle after a response or cancellation. */ + ctx->action = FD_FAILOVER_ACTION_IDLE; + ctx->stuck = failed; + fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); + fd_failover_channel_hangup( ctx->channel, now ); + sync_session( ctx ); +} + +/* An idle machine that does not run the identity authenticates to the + one that does ahead of an operator command. Merely pairing never + creates a request or changes the installed identity. */ +static void +preconnect( fd_failover_tile_ctx_t * ctx, + long now ) { + if( ctx->request.id || ctx->stuck || ctx->tx.valid || ctx->session.close_after_send || + ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) return; + /* A stale gossip address may name a validator that does not run the + identity any more. Do not keep that session open. Retain the dial + backoff. */ + if( !ctx->holds && + fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED && + !fd_failover_channel_peer_hello( ctx->channel )->holds ) + fd_failover_channel_hangup( ctx->channel, now ); + int ready = !ctx->holds && + ctx->staked_addr && ctx->staked_seen_at && now>=ctx->staked_seen_at && + now-ctx->staked_seen_atchannel, 0U, 0 ); + else if( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ) + fd_failover_channel_init_dialer( ctx->channel, ctx->staked_addr, FD_FAILOVER_DEFAULT_PORT ); +} + +/* An operator command is required before sending a handoff request. */ +static void +peer_poll( fd_failover_tile_ctx_t * ctx, + long now, + int * charge_busy ) { + preconnect( ctx, now ); + sync_session( ctx ); + if( FD_UNLIKELY( ctx->request.id && ctx->request.until && now>=ctx->request.until && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || + ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) ) { + /* The network deadline pauses this request without cancelling it. */ + request_pause( ctx, now, !ctx->request.peer.boot_id ? "no peer authenticated before the deadline" : + !ctx->request.send_cnt ? "no handoff request sent before the deadline" : + "no final answer before the deadline" ); + return; + } + if( FD_UNLIKELY( ctx->session.close_after_send && !ctx->tx.valid && !fd_failover_channel_tx_pending( ctx->channel ) ) ) { + /* The result drained, close the session without changing role or action. */ + ctx->session.close_after_send = 0; + fd_failover_channel_hangup( ctx->channel, now ); + FD_LOG_NOTICE(( "handoff %lu: confirmation drained, connection closed, listener ready for the next request", ctx->session.close_handoff_id )); + sync_session( ctx ); + return; + } + + ushort type; + ulong payload_sz; + if( FD_UNLIKELY( fd_failover_channel_poll( ctx->channel, now, charge_busy, &type, ctx->rx, &payload_sz ) ) ) { + sync_session( ctx ); + handle_control( ctx, type, payload_sz, now ); + } + sync_session( ctx ); + if( FD_LIKELY( !ctx->request.id || fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ) ) return; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + /* The peer restarted before it confirmed our answer, so it can no + longer confirm this handoff and the wait ends. A paused request + ends too, else the old holder's next request is refused as in + progress until the operator runs `failover promote` here. */ + if( FD_UNLIKELY( ctx->request.peer.boot_id && !peer_eq( &ctx->request.peer, peer->boot_id, peer->junk_pubkey ) && + ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && + ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ) ) { + int holds = ctx->holds; + int peer_holds = !!peer->holds; + FD_LOG_WARNING(( "handoff %lu: the peer restarted before it confirmed our %s, ending the request, %s, %s", ctx->request.id, + holds ? "ACK" : "refusal", holds ? "we run the identity" : "we do not run the identity", + holds && peer_holds ? "and its new boot says it runs the identity too, check `failover status` on both machines at once" : + holds ? "its new boot does not run the identity" : + peer_holds ? "its new boot runs the identity" : + "its new boot does not run it either, nothing votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both" )); + request_end( ctx, now, holds ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ); + return; + } + if( FD_LIKELY( !ctx->request.until || ctx->request.sent ) ) return; + if( FD_UNLIKELY( !ctx->request.peer.boot_id ) ) { + if( FD_UNLIKELY( !peer->holds ) ) { + FD_LOG_WARNING(( "handoff %lu dialed `" FD_IP4_ADDR_FMT ":%hu`, which does not run the identity, give the address of the machine running it with `failover promote --address`, " + "and if both machines show `runs target identity: no` no machine runs it and `failover promote --force` on the one whose status shows a coverage floor, never both, takes it", + ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), dial_port( ctx ) )); + /* End the peer wait without changing the installed identity. */ + request_end( ctx, now, FD_FAILOVER_HANDOFF_NOT_HOLDER ); + return; + } + /* Bind the request to the first authenticated member and boot running + the identity. */ + ctx->request.peer.boot_id = peer->boot_id; + fd_memcpy( ctx->request.peer.junk, peer->junk_pubkey, 32UL ); + } else if( FD_UNLIKELY( !peer_eq( &ctx->request.peer, peer->boot_id, peer->junk_pubkey ) ) ) { + /* Keep the old binding until the operator ends this request, without + accepting another boot's final state or acknowledgement for it. */ + ctx->request.peer_restarted = 1; + request_pause( ctx, now, "authenticated member or boot changed" ); + return; + } + fd_failover_handoff_request_t request = { .handoff_id=ctx->request.id, .target_boot_id=ctx->request.peer.boot_id, .replay_slot=ctx->replay_slot }; + if( FD_LIKELY( !fd_failover_channel_send( ctx->channel, now, FD_FAILOVER_MSG_HANDOFF_REQUEST, + (uchar const *)&request, sizeof(request) ) ) ) { + /* The request is sent, keep waiting for final state or confirmation. */ + ctx->request.sent = 1; + ctx->request.send_cnt = fd_ulong_sat_add( ctx->request.send_cnt, 1UL ); + if( now>=ctx->request.log_at ) { + ctx->request.log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); + char member[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( ctx->request.peer.junk, NULL, member ); + FD_LOG_NOTICE(( "handoff %lu: sent REQUEST to authenticated member %s boot %016lx, %s (send %lu)", + ctx->request.id, member, ctx->request.peer.boot_id, request_wait( ctx ), ctx->request.send_cnt )); + } + *charge_busy = 1; + } +} + + +/* Checks whether a promotion may go ahead now. Returns + FD_ADMINCTL_RESULT_SUCCESS or the refusal. */ +static ulong +promote_guard( fd_failover_tile_ctx_t const * ctx, + long now, + int force ) { + if( FD_UNLIKELY( ctx->input_lost ) ) return FD_FAILOVER_CONTROL_RESULT_INPUT_LOST; + int paired = fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED; + + if( FD_UNLIKELY( ctx->holds ) ) return FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER; + if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK && !force ) ) return FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING; + int request_wait = ctx->request.id && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || + ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + if( FD_UNLIKELY( ( ctx->action!=FD_FAILOVER_ACTION_IDLE && ctx->action!=FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK && + !( force && request_wait ) ) || + ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; + if( FD_UNLIKELY( force ) ) return FD_ADMINCTL_RESULT_SUCCESS; + /* The peer took the identity in our last handoff. */ + if( FD_UNLIKELY( ctx->taken ) ) return FD_FAILOVER_CONTROL_RESULT_TAKEN; + if( FD_UNLIKELY( paired && fd_failover_channel_peer_hello( ctx->channel )->holds ) ) + return FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS; + /* Another host is publishing the identity in gossip. */ + if( FD_UNLIKELY( ctx->staked_seen_at && now>=ctx->staked_seen_at && + now-ctx->staked_seen_ataction==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) return FD_FAILOVER_SOURCE_EMPTY; + ulong floor = coverage_floor( ctx ); + ulong root = ctx->root_slot; + int peer_ok = ctx->peer_history.valid && ( floor==FD_FAILOVER_SLOT_NULL || ctx->peer_history.tip>=floor ) && + ( root ==FD_FAILOVER_SLOT_NULL || ctx->peer_history.tip> root ); + return peer_ok ? FD_FAILOVER_SOURCE_PEER : FD_FAILOVER_SOURCE_EMPTY; +} + +/* Whether our handoff request is paused at its deadline. */ +static inline int +request_paused( fd_failover_tile_ctx_t const * ctx ) { + return ctx->request.id && !ctx->request.until && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); +} + +/* `failover promote` without --force: ask the machine running the + identity to hand it over, or resume a paused request. */ +static ulong +control_cmd_handoff( fd_failover_tile_ctx_t * ctx, + fd_adminctl_failover_req_t const * req, + long now ) { + int paused = request_paused( ctx ) && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT; + if( FD_UNLIKELY( paused && ctx->request.peer_restarted ) ) { + if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) { + /* Local adoption is already settled. This command ends only + our wait, without confirming anything on the old peer. */ + FD_LOG_WARNING(( "handoff %lu: the authenticated peer changed before it confirmed our answer, ending the confirmation wait, %s, the old peer's acknowledgement remains unknown", + ctx->request.id, ctx->holds ? "we run the identity" : "we do not run the identity" )); + request_end( ctx, now, ctx->holds ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ); + return FD_ADMINCTL_RESULT_SUCCESS; + } + FD_LOG_NOTICE(( "handoff %lu was bound to peer boot %016lx, the authenticated peer changed, ending it and starting a new handoff", ctx->request.id, ctx->request.peer.boot_id )); + request_end( ctx, now, FD_FAILOVER_HANDOFF_RESTARTED ); + paused = 0; + } + if( FD_UNLIKELY( paused ) ) { + /* Refresh the destination while retaining the authenticated member + and boot. An explicit address takes precedence over gossip. */ + ctx->request.until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); + ctx->request.sent = 0; + ctx->stuck = 0; + if( req->addr ) ctx->cmd_addr = req->addr; + if( req->port ) ctx->cmd_port = req->port; + if( ctx->cmd_addr ) ctx->request.addr = ctx->cmd_addr; + else if( ctx->staked_addr ) ctx->request.addr = ctx->staked_addr; + fd_failover_channel_init_dialer( ctx->channel, ctx->request.addr, dial_port( ctx ) ); + FD_LOG_NOTICE(( "resuming handoff %lu at `" FD_IP4_ADDR_FMT ":%hu` from %s, %s, %s", + ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), dial_port( ctx ), + ctx->cmd_addr ? "`failover promote --address`" : "gossip", + ctx->request.peer.boot_id ? "same authenticated member and boot" : "the other machine has not authenticated yet", + request_wait( ctx ) )); + return FD_ADMINCTL_RESULT_SUCCESS; + } + if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) + return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; + if( FD_UNLIKELY( ctx->holds ) ) return FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER; + uint addr = req->addr ? req->addr : ctx->staked_addr; + if( FD_UNLIKELY( !addr ) ) return FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS; + ctx->cmd_addr = req->addr; + ctx->cmd_port = req->port; + fd_memset( &ctx->request, 0, sizeof(ctx->request) ); + ctx->request.id = ctx->handoff_base+(++ctx->handoff_cnt); + if( FD_UNLIKELY( !ctx->request.id ) ) ctx->request.id = ++ctx->handoff_cnt; + ctx->request.last_id = ctx->request.id; + ctx->request.result = FD_FAILOVER_HANDOFF_PENDING; + ctx->request.addr = addr; + ctx->request.until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); + ctx->last_requested = 1; + ctx->stuck = 0; + /* Start a new handoff and wait for the machine running the identity. */ + ctx->action = FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER; + ctx->session.close_after_send = 0; + fd_failover_channel_init_dialer( ctx->channel, addr, dial_port( ctx ) ); + FD_LOG_NOTICE(( "requesting handoff %lu from the machine running the identity at `" FD_IP4_ADDR_FMT ":%hu` using %s, authenticating the peer before requesting final state", + ctx->request.id, FD_IP4_ADDR_FMT_ARGS( addr ), dial_port( ctx ), ctx->cmd_addr ? "`failover promote --address`" : "gossip" )); + return FD_ADMINCTL_RESULT_SUCCESS; +} + +/* `failover promote --force`: take the identity without the peer. */ +static ulong +control_cmd_promote( fd_failover_tile_ctx_t * ctx, + fd_adminctl_failover_req_t const * req, + long now ) { + /* --force takes over without dialing the peer and accepts incomplete + or empty history. */ + int force = !!( req->flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ); + /* Admin sends an unforced promote as a handoff, so this one did not + come through it. */ + if( FD_UNLIKELY( !force ) ) { + FD_LOG_WARNING(( "refusing a promote without --force, it did not come through admin" )); + return FD_ADMINCTL_RESULT_UNKNOWN_COMMAND; + } + ulong result = promote_guard( ctx, now, force ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) return result; + ulong source = promote_source( ctx ); + /* The operator fenced the peer. Only a request with no local + adoption or key switch in flight can be cancelled for recovery. */ + if( force && !ctx->holds && ctx->request.id && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) && + ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ) { + FD_LOG_NOTICE(( "handoff %lu cancelled by `failover promote --force` on the operator's word that the peer cannot sign, proceeding with local recovery", ctx->request.id )); + /* Forced recovery ends the peer wait without changing the installed identity. */ + request_end( ctx, now, FD_FAILOVER_HANDOFF_CANCELLED ); + } + if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) { + FD_LOG_WARNING(( "`failover promote --force` stops waiting for the peer's answer to handoff %lu, discarding saved final state because the peer may have voted, coverage floors retained", ctx->handoff.id )); + ctx->peer_history.valid = 0; + /* Forced recovery ends the ACK wait before starting local promotion. */ + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_CANCELLED ); + } + if( force ) FD_LOG_WARNING(( "operator promotion: --force asserts the peer cannot sign, peer checks are bypassed and incomplete or empty vote history is permitted if needed" )); + if( source==FD_FAILOVER_SOURCE_EMPTY ) { + fd_failover_history_t const * saved = &ctx->peer_history; + char const * stored = "the stored peer vote history"; + if( saved->valid ) { + ulong floor = coverage_floor( ctx ); + if( floor!=FD_FAILOVER_SLOT_NULL && saved->tiptip, floor )); + else if( ctx->root_slot!=FD_FAILOVER_SLOT_NULL && saved->tip<=ctx->root_slot ) + FD_LOG_NOTICE(( "operator promotion: %s ends at slot %lu at or below the local root %lu, skipping it", stored, saved->tip, ctx->root_slot )); + } + FD_LOG_NOTICE(( "operator promotion: no eligible saved final state is available, selecting an empty history" )); + } else { + fd_failover_history_t const * saved = &ctx->peer_history; + FD_LOG_NOTICE(( "operator promotion: selecting %s through slot %lu, the %s must validate and adopt it before the identity is installed", + source_name( source ), saved->tip, "votor" )); + } + /* The accepted operator command starts the replay and adoption sequence. */ + start_promotion( ctx, source, NULL, 0UL, force ); + return FD_ADMINCTL_RESULT_SUCCESS; +} + +/* Run an operator command. If we refuse it we say why. */ +static ulong +apply_control( fd_failover_tile_ctx_t * ctx, + fd_adminctl_failover_req_t const * req, + long now ) { + if( FD_UNLIKELY( ctx->input_lost ) ) return FD_FAILOVER_CONTROL_RESULT_INPUT_LOST; + switch( req->cmd ) { + case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: return control_cmd_handoff( ctx, req, now ); + case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: return control_cmd_promote( ctx, req, now ); + default: return FD_ADMINCTL_RESULT_UNSUPPORTED; + } +} + +/* The `failover status` response. */ +static void +status_snapshot( fd_failover_tile_ctx_t const * ctx, + long now, + fd_adminctl_failover_status_resp_t * resp ) { + fd_adminctl_failover_status_resp_init( resp ); + resp->enabled = 1; + resp->holds = (uchar)ctx->holds; + resp->action = (uchar)ctx->action; + resp->stuck = (uchar)!!ctx->stuck; + resp->link_state = (uchar)fd_failover_channel_state( ctx->channel ); + if( FD_UNLIKELY( fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED ) ) { + resp->peer_holds_valid = 1; + resp->peer_holds = (uchar)ctx->session.peer_holds; + } + resp->peer_boot_id = ctx->session.peer_boot_id; + resp->peer_addr = ctx->request.id ? ctx->request.addr : ctx->staked_addr; + resp->peer_addr_cmd = (uchar)( ctx->request.id && ctx->cmd_addr ); + resp->peer_port = ctx->request.id ? dial_port( ctx ) : FD_FAILOVER_DEFAULT_PORT; + resp->handoff_id = ctx->last_requested ? ctx->request.last_id : ctx->handoff.id; + resp->handoff_result = (uchar)( ctx->last_requested ? ctx->request.result : ctx->handoff.result ); + resp->promote_source = (uchar)promote_source( ctx ); + resp->promote_floor = coverage_floor( ctx ); + resp->promote_result = ctx->input_lost ? FD_FAILOVER_CONTROL_RESULT_INPUT_LOST : promote_guard( ctx, now, 0 ); + resp->mode = ctx->hello.mode; + resp->request_paused = (uchar)request_paused( ctx ); + fd_memcpy( resp->target, ctx->hello.staked_pubkey, 32UL ); + fd_memcpy( resp->installed, ctx->installed, 32UL ); + fd_memcpy( resp->junk, ctx->hello.junk_pubkey, 32UL ); +} + +/* Respond to one bus request. The admin tile validated the ABI already. + The response goes out in its own chunk. */ +static void +serve_bus_request( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem, + long now ) { + fd_adminctl_failover_req_t req; + fd_memcpy( &req, ctx->bus_req.payload, sizeof(req) ); + fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->admin_out_mem, ctx->admin_out_chunk ); + fd_memset( out, 0, sizeof(*out) ); + out->nonce = ctx->bus_req.nonce; + + if( FD_UNLIKELY( req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ) ) { + fd_adminctl_failover_status_resp_t status; + status_snapshot( ctx, now, &status ); + out->result = FD_ADMINCTL_RESULT_SUCCESS; + fd_memcpy( out->payload, &status, sizeof(status) ); + } else { + char const * cmd_name = fd_adminctl_failover_cmd_typed( req.cmd ); + FD_LOG_NOTICE(( "`%s%s%s` received", cmd_name, + ( req.flags & FD_ADMINCTL_FAILOVER_FLAG_YES ) ? " --yes" : "", + ( req.flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ) ? " --force" : "" )); + ulong result = apply_control( ctx, &req, now ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + char const * hint; + char const * name = control_refusal( result, &hint ); + FD_LOG_WARNING(( "`%s` refused with %s, %s", cmd_name, name, hint )); + } + ulong handoff_id = 0UL; + if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) { + /* Ending a paused confirmation wait zeroes request.id and keeps + last_id, which is the handoff the command acted on. */ + if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) handoff_id = ctx->request.id ? ctx->request.id : ctx->request.last_id; + } + fd_adminctl_failover_control_resp_t response = { + .version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, + .holds = (uchar)ctx->holds, + .action = (uchar)ctx->action, + .handoff_id = handoff_id, + }; + out->result = result; + fd_memcpy( out->payload, &response, sizeof(response) ); + } + + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_RESPONSE, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub ); + ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark ); +} + +/* The address of the machine running the identity comes from gossip, + from the newest contact info for the identity that is not ours. */ +static void +on_contact_info_msg( fd_failover_tile_ctx_t * ctx, + long now ) { + uint addr = ctx->gossip_socket.addr; + if( FD_UNLIKELY( !addr ) ) return; + if( FD_LIKELY( !fd_memeq( ctx->gossip_origin, ctx->hello.staked_pubkey, 32UL ) ) ) return; + if( FD_UNLIKELY( addr==ctx->own_gossip.addr && ctx->gossip_socket.port==ctx->own_gossip.port ) ) return; + ctx->staked_seen_at = now; + /* Gossip may be the only sign of a peer tenure, so we drop the peer's + history for good, not only while the contact info is fresh. */ + if( FD_UNLIKELY( ctx->peer_history.valid ) ) + FD_LOG_NOTICE(( "gossip shows the identity at another host, discarding saved final state from before its voting tenure, keeping coverage floors" )); + ctx->peer_history.valid = 0; + + if( FD_LIKELY( addr==ctx->staked_addr ) ) return; + uint old_addr = ctx->staked_addr; + ctx->staked_addr = addr; + if( FD_LIKELY( !old_addr ) ) { + FD_LOG_NOTICE(( "gossip shows the identity at `" FD_IP4_ADDR_FMT "`, a handoff from this machine dials it", FD_IP4_ADDR_FMT_ARGS( addr ) )); + } else { + FD_LOG_NOTICE(( "gossip shows a new address for the identity, `" FD_IP4_ADDR_FMT "` before and `" FD_IP4_ADDR_FMT "` now, the next handoff uses the new address", + FD_IP4_ADDR_FMT_ARGS( old_addr ), FD_IP4_ADDR_FMT_ARGS( addr ) )); + } +} + +/* Failover must not backpressure validator work. Losing an input instead + disables transitions because cached history or switch results may be stale. */ +static void +input_overrun( fd_failover_tile_ctx_t * ctx ) { + if( FD_UNLIKELY( ctx->input_lost ) ) return; + ctx->input_lost = 1; + ctx->stuck = 1; + ctx->hist_fresh = 0; + ctx->bus_req_fresh = 0; + fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); + fd_failover_channel_hangup( ctx->channel, fd_failover_clock() ); + FD_LOG_WARNING(( "failover input overrun, failover is disabled until restart, validator processing continues" )); +} + +static inline int +before_frag( fd_failover_tile_ctx_t * ctx, + ulong in_idx, + ulong seq FD_PARAM_UNUSED, + ulong sig ) { + if( FD_UNLIKELY( ctx->input_lost ) ) return in_idx!=ctx->admin_in_idx || sig!=FD_FAILOVER_BUS_REQUEST; + if( FD_LIKELY( in_idx==ctx->gossip_in_idx ) ) { + return sig!=FD_GOSSIP_UPDATE_TAG_CONTACT_INFO && sig!=FD_GOSSIP_UPDATE_TAG_CONTACT_INFO_REMOVE; + } + if( FD_LIKELY( in_idx==ctx->hist_in_idx ) ) { + /* Every history frame counts toward the drain watermark after the + stem validates it. */ + return 0; + } + if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { + /* Commands, switch responses and set-identity. Dropping a switch + response here would leave the switch hanging forever. */ + return sig!=FD_FAILOVER_BUS_REQUEST && sig!=FD_FAILOVER_BUS_SWITCH_RESP && sig!=FD_FAILOVER_BUS_OPERATOR_ACTION; + } + return 0; +} + +static void +during_frag( fd_failover_tile_ctx_t * ctx, + ulong in_idx, + ulong seq FD_PARAM_UNUSED, + ulong sig, + ulong chunk, + ulong sz, + ulong ctl FD_PARAM_UNUSED ) { + ctx->read_pending = 1; + if( FD_UNLIKELY( in_idx==ctx->adopt_in_idx ) ) { + if( FD_UNLIKELY( chunkadopt_in_chunk0 || chunk>ctx->adopt_in_wmark || sz!=sizeof(fd_votor_adopt_result_t) ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->adopt_in_chunk0, ctx->adopt_in_wmark )); + } + fd_memcpy( &ctx->adopt_result, fd_chunk_to_laddr_const( ctx->adopt_in_mem, chunk ), sizeof(fd_votor_adopt_result_t) ); + return; + } + if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { + if( FD_UNLIKELY( chunkadmin_in_chunk0 || chunk>ctx->admin_in_wmark || sz!=sizeof(fd_failover_bus_msg_t) ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->admin_in_chunk0, ctx->admin_in_wmark )); + } + fd_failover_bus_msg_t const * msg = fd_chunk_to_laddr_const( ctx->admin_in_mem, chunk ); + if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_SWITCH_RESP ) ) { + fd_memcpy( &ctx->id_switch.response, msg->payload, sizeof(ctx->id_switch.response) ); + ctx->id_switch.response_nonce = msg->nonce; + return; + } + if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_OPERATOR_ACTION ) ) { + fd_memcpy( &ctx->bus_operator, msg->payload, sizeof(ctx->bus_operator) ); + return; + } + fd_memcpy( &ctx->bus_req, msg, sizeof(fd_failover_bus_msg_t) ); + return; + } + if( FD_UNLIKELY( in_idx==ctx->hist_in_idx ) ) { + ulong frame_sz = sizeof(fd_votor_hist_msg_t); + if( FD_UNLIKELY( chunkhist_in_chunk0 || chunk>ctx->hist_in_wmark || sz!=frame_sz ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->hist_in_chunk0, ctx->hist_in_wmark )); + } + fd_memcpy( &ctx->hist, fd_chunk_to_laddr_const( ctx->hist_in_mem, chunk ), sizeof(fd_votor_hist_msg_t) ); + return; + } + if( FD_UNLIKELY( in_idx!=ctx->gossip_in_idx ) ) return; + if( FD_UNLIKELY( chunkgossip_in_chunk0 || chunk>ctx->gossip_in_wmark || sz>ctx->gossip_in_mtu ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->gossip_in_chunk0, ctx->gossip_in_wmark )); + } + /* Copy here and commit in after_frag. A remove leaves the socket + zero. */ + fd_gossip_update_message_t const * msg = fd_chunk_to_laddr_const( ctx->gossip_in_mem, chunk ); + fd_memcpy( ctx->gossip_origin, msg->origin, 32UL ); + ctx->gossip_socket.addr = 0U; + ctx->gossip_socket.port = (ushort)0; + if( FD_LIKELY( sig==FD_GOSSIP_UPDATE_TAG_CONTACT_INFO ) ) { + fd_gossip_socket_t const * socket = &msg->contact_info->value->sockets[ FD_GOSSIP_CONTACT_INFO_SOCKET_GOSSIP ]; + ctx->gossip_socket.addr = fd_uint_if ( !socket->is_ipv6, socket->ip4, 0U ); + ctx->gossip_socket.port = fd_ushort_if( !socket->is_ipv6, socket->port, (ushort)0 ); + } +} + +/* Unknown adoption results are invalid. */ +static ulong +votor_adopt_code( ulong result ) { + return resultread_pending = 0; + if( FD_UNLIKELY( in_idx==ctx->adopt_in_idx ) ) { + ctx->adopt_result.result = votor_adopt_code( ctx->adopt_result.result ); + ctx->adopt_result_id = sig; + ctx->adopt_result_fresh = 1; + return; + } + if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { + if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_SWITCH_RESP ) ) { + /* The drain still reads the last switch result, so we copy a + response into it only after the stem and nonce checks pass. */ + if( FD_LIKELY( on_switch_resp_msg( ctx, ctx->id_switch.response_nonce ) ) ) ctx->id_switch.result = ctx->id_switch.response; + if( FD_UNLIKELY( ctx->id_switch.response.result==FD_FAILOVER_SWITCH_ERR_STALE ) ) on_operator_action_msg( ctx, &ctx->id_switch.response.operator ); + return; + } + if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_OPERATOR_ACTION ) ) { + on_operator_action_msg( ctx, &ctx->bus_operator ); + return; + } + /* Send the response from after_credit, where a publish credit is available. */ + ctx->bus_req_fresh = 1; + return; + } + if( FD_UNLIKELY( in_idx==ctx->hist_in_idx ) ) { + ctx->hist_fresh = 1; + ctx->hist_seen_seq = seq; + return; + } + if( FD_UNLIKELY( in_idx!=ctx->gossip_in_idx ) ) return; + on_contact_info_msg( ctx, fd_failover_clock() ); +} + +static inline void +after_credit( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem, + int * opt_poll_in FD_PARAM_UNUSED, + int * charge_busy ) { + /* STEM skips after_frag if the fragment was overwritten during the read. */ + if( FD_UNLIKELY( ctx->read_pending ) ) input_overrun( ctx ); + if( FD_UNLIKELY( ctx->input_lost ) ) { + if( FD_UNLIKELY( ctx->bus_req_fresh ) ) { + ctx->bus_req_fresh = 0; + serve_bus_request( ctx, stem, fd_failover_clock() ); + *charge_busy = 1; + } + return; + } + if( FD_UNLIKELY( ctx->hist_fresh ) ) { + ctx->hist_fresh = 0; + on_votor_hist_msg( ctx, &ctx->hist ); + } + step_controller( ctx, stem ); + + long now = fd_failover_clock(); + if( FD_LIKELY( now>=ctx->poll_at ) ) { + int busy = 0; + peer_poll( ctx, now, &busy ); + ctx->poll_at = busy ? now : now+FD_FAILOVER_TILE_POLL_NANOS; + if( busy ) *charge_busy = 1; + } + pending_flush( ctx, now ); + if( FD_UNLIKELY( ctx->bus_req_fresh ) ) { + ctx->bus_req_fresh = 0; + serve_bus_request( ctx, stem, now ); + *charge_busy = 1; + } + + if( FD_UNLIKELY( ctx->stuck!=ctx->stuck_logged ) ) { + ctx->stuck_logged = ctx->stuck; + if( !ctx->stuck ) FD_LOG_NOTICE(( "failover is no longer stuck, %s", ctx->holds ? "this machine runs the identity" : "this machine does not run the identity" )); + } +} + +static ulong +populate_allowed_seccomp( fd_topo_t const * topo, + fd_topo_tile_t const * tile, + ulong out_cnt, + struct sock_filter * out ) { + fd_failover_tile_ctx_t const * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id ); + populate_sock_filter_policy_fd_failover_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)fd_failover_channel_listen_fd( ctx->channel ) ); + return sock_filter_policy_fd_failover_tile_instr_cnt; +} + +static ulong +rlimit_file_cnt( fd_topo_t const * topo FD_PARAM_UNUSED, + fd_topo_tile_t const * tile FD_PARAM_UNUSED ) { + return FD_FAILOVER_CHANNEL_CANDIDATE_MAX + 1UL + 8UL; +} + +static ulong +populate_allowed_fds( fd_topo_t const * topo, + fd_topo_tile_t const * tile, + ulong out_fds_cnt, + int * out_fds ) { + fd_failover_tile_ctx_t const * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id ); + int logfile_fd = fd_log_private_logfile_fd(); + int listen_fd = fd_failover_channel_listen_fd( ctx->channel ); + + ulong required_fds = 1UL + (ulong)(-1!=logfile_fd) + (ulong)(-1!=listen_fd); + if( FD_UNLIKELY( out_fds_cnt + +/* ALPN id, TLS length-prefixed */ +static uchar const ALPN[] = "\x17" "fd-failover/on-demand/1"; + +FD_STATIC_ASSERT( sizeof(ALPN)==25UL, alpn_size ); + +int +fd_failover_tls_ctx_init( fd_failover_tls_ctx_t * ctx, + uchar const public_key[ 32 ], + fd_tls_sign_t signer ) { + fd_failover_tls_ctx_fini( ctx ); + + uchar seed[ 32 ]; + if( FD_UNLIKELY( !fd_rng_secure( seed, 32UL ) ) ) return -1; + fd_chacha_rng_t * rng = fd_chacha_rng_join( fd_chacha_rng_new( ctx->rng, FD_CHACHA_RNG_MODE_MOD ) ); + FD_TEST( rng ); + fd_chacha_rng_init( rng, seed, FD_CHACHA_RNG_ALGO_CHACHA20 ); + fd_memzero_explicit( seed, sizeof(seed) ); + + fd_tls_t * tls = fd_tls_join( fd_tls_new( &ctx->tls ) ); + FD_TEST( tls ); + tls->rng = rng; + tls->sign = signer; + fd_memcpy( tls->cert_public_key, public_key, 32UL ); + fd_x509_mock_cert( tls->cert_x509, tls->cert_public_key ); + tls->cert_x509_sz = FD_X509_MOCK_CERT_SZ; + fd_memcpy( tls->alpn, ALPN, sizeof(ALPN)-1UL ); + tls->alpn_sz = sizeof(ALPN)-1UL; + tls->quic = 0; + tls->req_client_cert = 1; + + /* The TLS configuration is ready for new connections. */ + ctx->ready = 1; + return 0; +} + +void +fd_failover_tls_ctx_fini( fd_failover_tls_ctx_t * ctx ) { + /* Wiping the context prevents new connections until reinitialization. */ + fd_memzero_explicit( ctx, sizeof(*ctx) ); +} + +int +fd_failover_tls_new( fd_failover_tls_t * tls, + fd_failover_tls_ctx_t * ctx, + int fd, + int dial_peer ) { + if( FD_UNLIKELY( !ctx->ready ) ) return -1; + /* Start an unverified, unpaired connection. */ + fd_memset( tls, 0, sizeof(*tls) ); + tls->fd = fd; + fd_tlsrec_conn_init( &tls->conn, &ctx->tls, !dial_peer ); + /* Fresh X25519 key share per connection, straight from the kernel. */ + uchar seed[ 32 ]; + if( FD_UNLIKELY( !fd_rng_secure( seed, 32UL ) ) ) return -1; + fd_chacha_rng_init( ctx->rng, seed, FD_CHACHA_RNG_ALGO_CHACHA20 ); + fd_memzero_explicit( seed, sizeof(seed) ); + if( FD_UNLIKELY( !fd_rng_secure( tls->conn.tls.kex_private_key, 32UL ) ) ) return -1; + fd_x25519_public( tls->conn.tls.kex_public_key, tls->conn.tls.kex_private_key ); + fd_tlsrec_sock_init( &tls->sock ); + return 0; +} + +int +fd_failover_tls_client_hello( fd_failover_tls_t * tls, uchar * out, ulong * out_sz ) { + ulong app_sz = 0UL; + return fd_tlsrec_conn_rx( &tls->conn, NULL, out, out_sz, NULL, &app_sz ); +} + +int +fd_failover_tls_accept_flight( fd_failover_tls_t * tls, uchar const * data, ulong sz ) { + if( fd_tlsrec_sock_tx_pending( &tls->sock ) || tls->received>FD_FAILOVER_TLS_PREPAIR_MAX || + sz>FD_FAILOVER_TLS_PREPAIR_MAX-tls->received ) return -1; + fd_tlsrec_slice_t rx[1]; + fd_tlsrec_slice_init( rx, (uchar *)data, sz ); + ulong tx_sz = sizeof(tls->sock.tx_buf); + ulong app_sz = 0UL; + int rc = fd_tlsrec_conn_rx( &tls->conn, rx, tls->sock.tx_buf, &tx_sz, NULL, &app_sz ); + tls->sock.tx_sz = tx_sz; + tls->received += sz; + return rc || !fd_tlsrec_slice_is_empty( rx ) || app_sz ? -1 : 0; +} + +void +fd_failover_tls_fini( fd_failover_tls_t * tls ) { + if( FD_LIKELY( tls->fd!=-1 && fd_tlsrec_conn_is_ready( &tls->conn ) && !tls->conn.tx_closed ) ) { + (void)fd_tlsrec_sock_close( &tls->sock, &tls->conn, tls->fd ); + } + fd_memzero_explicit( &tls->conn, sizeof(tls->conn) ); + fd_memzero_explicit( &tls->sock, sizeof(tls->sock) ); + tls->fd = -1; + tls->verified = 0; + tls->paired = 0; + tls->received = 0UL; +} + +static int +pull( fd_failover_tls_t * tls ) { + if( !tls->paired && tls->received>=FD_FAILOVER_TLS_PREPAIR_MAX ) return -1; + ulong rx_sz = 0UL; + int err = fd_tlsrec_sock_rx( &tls->sock, &tls->conn, tls->fd, &rx_sz ); + if( !tls->paired ) tls->received += rx_sz; + if( FD_LIKELY( !err ) ) return 0; + if( err==FD_TLSREC_SOCK_ERR_EOF ) tls->peer_closed = 1; + return -1; +} + +int +fd_failover_tls_handshake( fd_failover_tls_t * tls ) { + if( FD_LIKELY( tls->verified ) ) return 1; /* Verification is final for this connection. */ + /* A failed handshake stays failed until the channel closes it. */ + if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) ) ) { + FD_LOG_DEBUG(( "failover TLS handshake failed: %s", fd_tls_reason_cstr( tls->conn.hs.base.reason ) )); + return -1; + } + if( FD_UNLIKELY( fd_tlsrec_sock_flush( &tls->sock, tls->fd )<0 ) ) return -1; + if( FD_LIKELY( !fd_tlsrec_conn_is_ready( &tls->conn ) ) ) { + if( FD_UNLIKELY( pull( tls ) ) ) return -1; + if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) ) ) { + FD_LOG_DEBUG(( "failover TLS handshake failed: %s", fd_tls_reason_cstr( tls->conn.hs.base.reason ) )); + return -1; + } + if( FD_LIKELY( !fd_tlsrec_conn_is_ready( &tls->conn ) ) ) return 0; /* Continue on a later poll. */ + } + + /* Handshake done, the peer signed with some key and agreed on our ALPN. + We keep that key, the channel checks it against HELLO. */ + + if( !fd_tlsrec_conn_is_server( &tls->conn ) ) { + fd_tls_estate_cli_t const * cli = &tls->conn.hs.cli; + if( FD_UNLIKELY( cli->server_key_type!=FD_TLS_KEY_ED25519 || + cli->server_pubkey_len!=32UL || !cli->alpn_negotiated ) ) return -1; + fd_memcpy( tls->peer_pubkey, cli->server_pubkey, 32UL ); + } else { + fd_tls_estate_srv_t const * srv = &tls->conn.hs.srv; + fd_memcpy( tls->peer_pubkey, srv->client_pubkey, 32UL ); + } + /* TLS verified the peer key, HELLO must still authorize the member. */ + tls->verified = 1; + return 1; +} + +long +fd_failover_tls_read( fd_failover_tls_t * tls, + void * buf, + ulong sz ) { + if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) ) ) return -1L; + if( FD_LIKELY( !fd_tlsrec_sock_rx_avail( &tls->sock ) ) ) { + if( FD_UNLIKELY( fd_tlsrec_sock_flush( &tls->sock, tls->fd )<0 ) ) return -1L; + if( FD_UNLIKELY( pull( tls ) ) ) return -1L; + } + return (long)fd_tlsrec_sock_rx_pop( &tls->sock, buf, sz ); +} + +long +fd_failover_tls_write( fd_failover_tls_t * tls, + void const * buf, + ulong sz ) { + if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) || tls->conn.tx_closed ) ) return -1L; + int rc = fd_tlsrec_sock_flush( &tls->sock, tls->fd ); + if( FD_UNLIKELY( rc ) ) return rc<0 ? -1L : 0L; /* Earlier output is still parked. */ + ulong consumed = 0UL; + if( FD_UNLIKELY( fd_tlsrec_sock_tx( &tls->sock, &tls->conn, tls->fd, buf, sz, &consumed ) ) ) return -1L; + return (long)consumed; +} diff --git a/src/discof/failover/fd_failover_tls.h b/src/discof/failover/fd_failover_tls.h new file mode 100644 index 00000000000..b6dfa83db26 --- /dev/null +++ b/src/discof/failover/fd_failover_tls.h @@ -0,0 +1,103 @@ +#ifndef HEADER_fd_src_discof_failover_fd_failover_tls_h +#define HEADER_fd_src_discof_failover_fd_failover_tls_h + +/* TLS 1.3 over TCP for one failover candidate, built on fd_tls, + fd_tlsrec and fd_tlsrec_sock. Both ends present an Ed25519 cert for + their junk identity. The channel admits a peer before TLS and checks + its TLS key against the admitted HELLO. The caller owns the socket. */ + +#include "../../waltz/tlsrec/fd_tlsrec_sock.h" +#include "../../ballet/chacha/fd_chacha_rng.h" + +/* Max ciphertext accepted from a candidate before it pairs */ +#define FD_FAILOVER_TLS_PREPAIR_MAX (32768UL) + +/* Per-channel TLS config, the fd_tls template and its RNG. The sign + tile signs our CertificateVerify, no private key is kept here. */ + +struct fd_failover_tls_ctx { + fd_tls_t tls; + fd_chacha_rng_t rng[ 1 ]; + int ready; /* set by init and wiped by fini, fd_failover_tls_new fails without it */ +}; +typedef struct fd_failover_tls_ctx fd_failover_tls_ctx_t; + +/* One TLS connection. */ + +struct fd_failover_tls { + fd_tlsrec_conn_t conn; + fd_tlsrec_sock_t sock; + int fd; + int verified; /* handshake done, peer_pubkey is set */ + int paired; /* set by the channel after HELLO */ + int peer_closed; /* orderly EOF or close_notify */ + uchar peer_pubkey[ 32 ]; /* junk key the peer signed the handshake with */ + ulong received; /* ciphertext bytes read before pairing */ +}; +typedef struct fd_failover_tls fd_failover_tls_t; + +FD_PROTOTYPES_BEGIN + +/* fd_failover_tls_ctx_init sets our junk pubkey and the signer for + CertificateVerify and seeds the RNG. It and every new connection read + entropy from getrandom. Returns 0 on success, -1 if entropy fails. */ + +int +fd_failover_tls_ctx_init( fd_failover_tls_ctx_t * ctx, + uchar const public_key[ 32 ], + fd_tls_sign_t signer ); + +/* fd_failover_tls_ctx_fini wipes the context. */ + +void +fd_failover_tls_ctx_fini( fd_failover_tls_ctx_t * ctx ); + +/* fd_failover_tls_new binds a fresh connection to fd. The listener + requires a client cert. Returns 0 on success, -1 if ctx is not + initialized or entropy fails. */ + +int +fd_failover_tls_new( fd_failover_tls_t * tls, + fd_failover_tls_ctx_t * ctx, + int fd, + int dial_peer ); + +/* Prepare the outbound ClientHello without sending it, or feed an + admitted peer flight to the connection without socket I/O. The + generated reply is left in sock.tx_buf for the caller to capture or + flush with fd_tlsrec_sock_flush. No earlier output may be pending. */ +int +fd_failover_tls_client_hello( fd_failover_tls_t * tls, uchar * out, ulong * out_sz ); + +int +fd_failover_tls_accept_flight( fd_failover_tls_t * tls, uchar const * data, ulong sz ); + +/* fd_failover_tls_fini sends close_notify if the connection is up and + wipes the connection state. The caller closes fd. */ + +void +fd_failover_tls_fini( fd_failover_tls_t * tls ); + +/* fd_failover_tls_handshake returns 1 once the handshake is done and + the peer signed it with an Ed25519 key, which is then in peer_pubkey, + 0 while pending, -1 on failure. fd_failover_tls_read and + fd_failover_tls_write return bytes moved, 0 when pending, -1 on error + or peer close. A write may take fewer bytes than offered (at most one + record), callers are responsible for retrying to obtain the rest. */ + +int +fd_failover_tls_handshake( fd_failover_tls_t * tls ); + +long +fd_failover_tls_read( fd_failover_tls_t * tls, + void * buf, + ulong sz ); + +long +fd_failover_tls_write( fd_failover_tls_t * tls, + void const * buf, + ulong sz ); + +FD_PROTOTYPES_END + +#endif /* HEADER_fd_src_discof_failover_fd_failover_tls_h */ diff --git a/src/discof/failover/fd_failover_wire.c b/src/discof/failover/fd_failover_wire.c new file mode 100644 index 00000000000..7e8915734a1 --- /dev/null +++ b/src/discof/failover/fd_failover_wire.c @@ -0,0 +1,65 @@ +#include "fd_failover_wire.h" + +#include + +static ulong const payload_max[ FD_FAILOVER_MSG_CNT ] = { + sizeof(fd_failover_hello_t), + sizeof(fd_failover_handoff_request_t), + FD_FAILOVER_DEMOTED_PAYLOAD_MAX, + sizeof(fd_failover_promote_ack_t), + sizeof(fd_failover_promote_rejected_t), + sizeof(fd_failover_handoff_result_t), + 0UL /* KEEPALIVE */ +}; + +ulong +fd_failover_wire_encode( uchar * out, + ushort type, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( type>=FD_FAILOVER_MSG_CNT || + payload_sz>payload_max[type] ) ) return 0UL; + + ulong frame_sz = FD_FAILOVER_FRAME_HDR_SZ+payload_sz; + + fd_failover_frame_hdr_t hdr = { + .version = (ushort)FD_FAILOVER_VERSION, + .len = (uint)frame_sz, + .type = type, + }; + fd_memcpy( out, &hdr, FD_FAILOVER_FRAME_HDR_SZ ); + if( FD_LIKELY( payload_sz ) ) fd_memcpy( out+FD_FAILOVER_FRAME_HDR_SZ, payload, payload_sz ); + + return frame_sz; +} + +int +fd_failover_wire_decode( uchar const * buf, + ulong buf_sz, + ushort * out_type, + uchar const ** out_payload, + ulong * out_payload_sz, + ulong * out_frame_sz ) { + if( FD_UNLIKELY( buf_szFD_FAILOVER_FRAME_MAX ) ) return FD_FAILOVER_WIRE_ERR_SZ; + + ulong payload_sz = frame_sz-FD_FAILOVER_FRAME_HDR_SZ; + + if( FD_UNLIKELY( hdr.version!=(ushort)FD_FAILOVER_VERSION ) ) return FD_FAILOVER_WIRE_ERR_VERSION; + if( FD_UNLIKELY( hdr.type>=FD_FAILOVER_MSG_CNT ) ) return FD_FAILOVER_WIRE_ERR_TYPE; + + if( FD_UNLIKELY( payload_sz>payload_max[hdr.type] ) ) return FD_FAILOVER_WIRE_ERR_SZ; + if( FD_UNLIKELY( buf_sz +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__i386__) +# define ARCH_NR AUDIT_ARCH_I386 +#elif defined(__x86_64__) +# define ARCH_NR AUDIT_ARCH_X86_64 +#elif defined(__aarch64__) +# define ARCH_NR AUDIT_ARCH_AARCH64 +#elif defined(__riscv) && __riscv_xlen == 64 +# define ARCH_NR AUDIT_ARCH_RISCV64 +#else +# error "Target architecture is unsupported by seccomp." +#endif + +#define FD_SECCOMP_ARG_LO_OFFSET(argno) ( offsetof( struct seccomp_data, args[(argno)] ) ) +#define FD_SECCOMP_ARG_HI_OFFSET(argno) ( offsetof( struct seccomp_data, args[(argno)] ) + 4U ) + +#define FD_SECCOMP_ARG_LO(x) ((uint)(((ulong)(uint)(int)(x) ) & 0xffffffffUL)) +#define FD_SECCOMP_ARG_HI(x) ((uint)(((ulong)(x) >> 32) & 0xffffffffUL)) + +static const uint sock_filter_policy_fd_failover_tile_instr_cnt = 53; + +static void populate_sock_filter_policy_fd_failover_tile( ulong out_cnt, struct sock_filter out[ static 53 ], uint logfile_fd, uint listen_fd ) { + FD_TEST( out_cnt >= 53 ); + struct sock_filter filter[53] = { + /* validate architecture */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, arch ) )), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 11 ), + /* load syscall number */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, nr ) )), + /* check write */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_write, /* check_write */ 11, 0 ), + /* check fsync */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_fsync, /* check_fsync */ 16, 0 ), + /* allow getrandom */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_getrandom, /* RET_ALLOW */ 8, 0 ), + /* check socket */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_socket, /* check_socket */ 18, 0 ), + /* allow connect */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_connect, /* RET_ALLOW */ 6, 0 ), + /* check setsockopt */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_setsockopt, /* check_setsockopt */ 24, 0 ), + /* check accept4 */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_accept4, /* check_accept4 */ 29, 0 ), + /* allow recvfrom */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_recvfrom, /* RET_ALLOW */ 3, 0 ), + /* allow sendto */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_sendto, /* RET_ALLOW */ 2, 0 ), + /* check close */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_close, /* check_close */ 32, 0 ), +// RET_KILL_PROCESS: + /* default deny */ + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// RET_ALLOW: + /* allow */ + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_write: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 0x00000002U, /* write_ALLOW */ 3, /* or_1 */ 0 ), +// or_1: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(logfile_fd)), /* write_ALLOW */ 1, /* write_KILL */ 0 ), +// write_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// write_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_fsync: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(logfile_fd)), /* fsync_ALLOW */ 1, /* fsync_KILL */ 0 ), +// fsync_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// fsync_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_socket: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(AF_INET), /* and_2 */ 0, /* socket_KILL */ 4 ), +// and_2: + /* arg 1 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(1)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC), /* and_3 */ 0, /* socket_KILL */ 2 ), +// and_3: + /* arg 2 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(2)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 0x00000000U, /* socket_ALLOW */ 1, /* socket_KILL */ 0 ), +// socket_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// socket_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_setsockopt: + /* arg 1 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(1)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(IPPROTO_TCP), /* and_4 */ 0, /* setsockopt_KILL */ 2 ), +// and_4: + /* arg 2 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(2)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(TCP_NODELAY), /* setsockopt_ALLOW */ 1, /* setsockopt_KILL */ 0 ), +// setsockopt_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// setsockopt_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_accept4: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(listen_fd)), /* and_5 */ 0, /* accept4_KILL */ 2 ), +// and_5: + /* arg 3 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(3)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(SOCK_NONBLOCK|SOCK_CLOEXEC), /* accept4_ALLOW */ 1, /* accept4_KILL */ 0 ), +// accept4_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// accept4_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_close: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 0x00000002U, /* close_KILL */ 4, /* or_6 */ 0 ), +// or_6: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(logfile_fd)), /* close_KILL */ 2, /* or_7 */ 0 ), +// or_7: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(listen_fd)), /* close_KILL */ 0, /* close_ALLOW */ 1 ), +// close_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// close_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), + }; + fd_memcpy( out, filter, sizeof( filter ) ); +} + +#endif /* defined(__linux__) */ + +#endif /* HEADER_fd_src_discof_failover_generated_fd_failover_tile_seccomp_h */ diff --git a/src/discof/failover/test_failover_channel.c b/src/discof/failover/test_failover_channel.c new file mode 100644 index 00000000000..ec11b20db03 --- /dev/null +++ b/src/discof/failover/test_failover_channel.c @@ -0,0 +1,109 @@ +#include "fd_failover_channel.h" +#include "../../disco/keyguard/fd_keyguard.h" +#include "../../ballet/ed25519/fd_ed25519.h" +#include "../../util/net/fd_ip4.h" + +static uchar scratch[ 2 ][ 32UL<<20 ] __attribute__((aligned(128))); +static uchar keys[ 3 ][ 64 ] = { { 1 }, { 2 }, { 9 } }; +static fd_sha512_t sha[ 1 ]; + +/* Stands in for the sign tile, the member's junk key signs its TLS. */ +static void +sign_fn( void * ctx, + uchar sig[ static FD_ED25519_SIG_SZ ], + uchar const payload[ static FD_TLS_CV_SIGN_SZ ] ) { + uchar const * key = ctx; + fd_ed25519_sign( sig, payload, FD_TLS_CV_SIGN_SZ, key+32UL, key, sha ); +} + +static void +proof_fn( void * ctx, uchar sig[ 64 ], uchar const msg[ FD_KEYGUARD_FAILOVER_PROOF_SZ ] ) { + uchar const * key = ctx; + fd_ed25519_sign( sig, msg, FD_KEYGUARD_FAILOVER_PROOF_SZ, key+32UL, key, sha ); +} + +static fd_failover_channel_t * +member( ulong idx, ulong staked_idx ) { + fd_failover_channel_t * ch = fd_failover_channel_join( fd_failover_channel_new( scratch[idx] ) ); + FD_TEST( ch ); + fd_failover_hello_t hello = { .version=FD_FAILOVER_VERSION, .boot_id=idx+1UL, + .holds=(uchar)(idx ? 0 : 1), + .mode=FD_FAILOVER_MODE_ALPENGLOW }; + fd_memcpy( hello.junk_pubkey, keys[idx]+32UL, 32UL ); + fd_memcpy( hello.staked_pubkey, keys[staked_idx]+32UL, 32UL ); + fd_memset( hello.vote_account, 0xBB, 32UL ); + FD_TEST( !fd_failover_channel_set_identity( ch, keys[idx]+32UL, (fd_tls_sign_t){ .ctx=keys[idx], .sign_fn=sign_fn }, + (fd_failover_proof_signer_t){ .ctx=keys[staked_idx], .sign_fn=proof_fn }, &hello ) ); + fd_failover_channel_init_listener( ch, FD_IP4_ADDR(127,0,0,1), 0 ); + return ch; +} + +static int +poll_one( fd_failover_channel_t * ch, ushort * type, uchar * payload, ulong * sz ) { + int busy = 0; + return fd_failover_channel_poll( ch, fd_failover_clock(), &busy, type, payload, sz ); +} + +static void +test_pair_and_transfer( void ) { + fd_failover_channel_t * a = member( 0UL, 2UL ); + fd_failover_channel_t * b = member( 1UL, 2UL ); + fd_failover_channel_init_dialer( b, FD_IP4_ADDR(127,0,0,1), fd_failover_channel_listen_port( a ) ); + long until = fd_failover_clock()+5000000000L; + ushort type; + ulong sz; + uchar payload[ FD_FAILOVER_PAYLOAD_MAX ]; + while( fd_failover_channel_state( a )!=FD_FAILOVER_SESSION_PAIRED || + fd_failover_channel_state( b )!=FD_FAILOVER_SESSION_PAIRED ) { + FD_TEST( fd_failover_clock()boot_id && !fd_failover_channel_peer_hello( b )->boot_id ); + fd_failover_channel_fini( a ); + fd_failover_channel_fini( b ); +} + +int +main( int argc, char ** argv ) { + fd_boot( &argc, &argv ); + FD_TEST( fd_failover_channel_footprint()<=sizeof(scratch[0]) ); + FD_TEST( fd_sha512_join( fd_sha512_new( sha ) ) ); + for( ulong i=0UL; i<3UL; i++ ) fd_ed25519_public_from_private( keys[i]+32UL, keys[i], sha ); + test_pair_and_transfer(); + test_wrong_staked_key(); + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} diff --git a/src/discof/failover/test_failover_proto.c b/src/discof/failover/test_failover_proto.c new file mode 100644 index 00000000000..ff3fe036940 --- /dev/null +++ b/src/discof/failover/test_failover_proto.c @@ -0,0 +1,94 @@ +#include "fd_failover_proto.h" +#include "../../choreo/votor/ag_hist.h" +#include "../../util/fd_util.h" + +static void +fill_hello( fd_failover_hello_t * h, + uchar junk, + uchar staked, + uchar vote, + uchar role ) { + fd_memset( h, 0, sizeof(fd_failover_hello_t) ); + h->version = (ushort)FD_FAILOVER_VERSION; + fd_memset( h->junk_pubkey, junk, 32UL ); + fd_memset( h->staked_pubkey, staked, 32UL ); + fd_memset( h->vote_account, vote, 32UL ); + h->holds = role; + h->mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; + h->boot_id = 0x1000UL+junk; +} + +static void +test_hello( void ) { + fd_failover_hello_t holder, other; + fill_hello( &holder, 1U, 0xAA, 0xBB, 1 ); + fill_hello( &other, 2U, 0xAA, 0xBB, 0 ); + FD_TEST( fd_failover_hello_check( &holder, &other )==FD_FAILOVER_HELLO_OK ); + FD_TEST( fd_failover_hello_check( &other, &holder )==FD_FAILOVER_HELLO_OK ); + uchar unsupported[] = { 0U, 2U, 255U }; + for( ulong i=0UL; imode = 0U; + FD_TEST( !fd_failover_demoted_decode( &decoded, payload, sz ) ); + header->mode = FD_FAILOVER_MODE_ALPENGLOW; + header->last_vote_slot = 102UL; + /* The wire layer does not read the history, the controller compares + the adopted tip with last_vote_slot. */ + FD_TEST( fd_failover_demoted_decode( &decoded, payload, sz ) && decoded.last_vote_slot==102UL ); +} + +/* test_handoff_decode: requests and results need their exact size and + nonzero ids. */ +static void +test_handoff_decode( void ) { + fd_failover_handoff_request_t request = { .handoff_id=5UL, .target_boot_id=6UL, .replay_slot=FD_FAILOVER_SLOT_NULL }, request_out; + FD_TEST( fd_failover_handoff_request_decode( &request_out, (uchar const *)&request, sizeof(request) ) && request_out.handoff_id==5UL ); + FD_TEST( request_out.replay_slot==FD_FAILOVER_SLOT_NULL ); + FD_TEST( !fd_failover_handoff_request_decode( &request_out, (uchar const *)&request, sizeof(request)-1UL ) ); + request.target_boot_id = 0UL; + FD_TEST( !fd_failover_handoff_request_decode( &request_out, (uchar const *)&request, sizeof(request) ) ); + + fd_failover_handoff_result_t result = { .handoff_id=5UL }, result_out; + FD_TEST( fd_failover_handoff_result_decode( &result_out, (uchar const *)&result, sizeof(result) ) && result_out.handoff_id==5UL ); + FD_TEST( !fd_failover_handoff_result_decode( &result_out, (uchar const *)&result, sizeof(result)+1UL ) ); + result.handoff_id = 0UL; + FD_TEST( !fd_failover_handoff_result_decode( &result_out, (uchar const *)&result, sizeof(result) ) ); +} + +int +main( int argc, char ** argv ) { + fd_boot( &argc, &argv ); + test_hello(); + test_demoted_decode(); + test_handoff_decode(); + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} diff --git a/src/discof/failover/test_failover_tile.c b/src/discof/failover/test_failover_tile.c new file mode 100644 index 00000000000..68068fb68a5 --- /dev/null +++ b/src/discof/failover/test_failover_tile.c @@ -0,0 +1,875 @@ +#include "fd_failover_channel.c" +#include "fd_failover_tile.c" +#include "../../ballet/ed25519/fd_ed25519.h" + +#include + +#define OWN_GOSSIP_ADDR FD_IP4_ADDR(10,0,0,1) +#define OWN_GOSSIP_PORT ((ushort)8001) + +static fd_frag_meta_t pub_mcache[ 8 ]; +static ulong pub_cr_avail; +static ulong pub_min_cr_avail; +static int pub_reliable; +static fd_stem_context_t stem[ 1 ]; +static uchar bus_mem[ 8192 ] __attribute__((aligned(128))); + +static void +stem_init( void ) { + static fd_frag_meta_t * mcaches[ 1 ]; + static ulong seqs[ 1 ]; + static ulong depths[ 1 ]; + mcaches[ 0 ] = pub_mcache; + seqs[ 0 ] = 0UL; + depths[ 0 ] = 8UL; + pub_cr_avail = 64UL; + pub_min_cr_avail = 64UL; + pub_reliable = 0; + *stem = (fd_stem_context_t){ + .mcaches = mcaches, .seqs = seqs, .depths = depths, + .cr_avail = &pub_cr_avail, .min_cr_avail = &pub_min_cr_avail, + .cr_decrement_amount = 1UL, .out_reliable = &pub_reliable, + }; +} + +static uchar ch_mem[ 32UL<<20 ] __attribute__((aligned(128))); +static fd_failover_tile_ctx_t ctl[ 1 ]; + +#define OUR_BOOT_ID (1000UL) + +static fd_failover_tile_ctx_t * +controller_init( int holds ) { + stem_init(); + fd_failover_tile_ctx_t * ctx = ctl; + fd_memset( ctx, 0, sizeof(fd_failover_tile_ctx_t) ); + ctx->holds = holds; + ctx->hello.holds = (uchar)holds; + ctx->hello.mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; + ctx->hello.boot_id = OUR_BOOT_ID; + fd_memset( ctx->hello.junk_pubkey, 0x11, 32UL ); + fd_memset( ctx->hello.staked_pubkey, 0x5A, 32UL ); + fd_memcpy( ctx->installed, holds ? ctx->hello.staked_pubkey : ctx->hello.junk_pubkey, 32UL ); + ctx->replay_slot = 100UL; + ctx->root_slot = 90UL; + ctx->last_vote_slot = 99UL; + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT; + ctx->deadline_slot = FD_FAILOVER_SLOT_NULL; + ctx->peer_floor = FD_FAILOVER_SLOT_NULL; + ctx->own_floor = FD_FAILOVER_SLOT_NULL; + ctx->adopt_anchor = FD_FAILOVER_SLOT_NULL; + ctx->empty_vote_after = FD_FAILOVER_SLOT_NULL; + ctx->hist_seen_seq = ULONG_MAX; + ctx->handoff_base = 5000UL; + ctx->gossip_in_idx = ULONG_MAX; + ctx->hist_in_idx = ULONG_MAX; + ctx->admin_in_idx = ULONG_MAX; + ctx->adopt_in_idx = ULONG_MAX; + ctx->admin_out_idx = 0UL; + ctx->admin_out_mem = (fd_wksp_t *)bus_mem; + ctx->adopt_out_idx = 0UL; + ctx->adopt_out_mem = (fd_wksp_t *)bus_mem; + ctx->own_gossip.addr = OWN_GOSSIP_ADDR; + ctx->own_gossip.port = fd_ushort_bswap( OWN_GOSSIP_PORT ); + ctx->channel = fd_failover_channel_join( fd_failover_channel_new( ch_mem ) ); + FD_TEST( ctx->channel ); + ctx->channel->self_hello = ctx->hello; + ctx->session.state = fd_failover_channel_state( ctx->channel ); + return ctx; +} + +static int fake_peer_fd = -1; + +static void +controller_fini( fd_failover_tile_ctx_t * ctx ) { + fd_failover_channel_fini( ctx->channel ); + if( fake_peer_fd!=-1 ) close( fake_peer_fd ); + fake_peer_fd = -1; +} + +static void +pair( fd_failover_tile_ctx_t * ctx, + ulong boot_id, + ulong peer_holds, + long now ) { + fd_failover_channel_t * ch = ctx->channel; + struct candidate * c = &ch->candidates[ ADMITTED_IDX ]; + if( c->fd==-1 ) { + int sv[ 2 ]; + FD_TEST( !socketpair( AF_UNIX, SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC, 0, sv ) ); + fd_memset( c, 0, sizeof(*c) ); + c->fd = sv[ 0 ]; + c->tls.fd = sv[ 0 ]; + c->phase = PHASE_READY; + c->tls.conn.hs.base.state = FD_TLS_HS_CONNECTED; + c->tls.paired = 1; + c->local_hello = ctx->hello; + fake_peer_fd = sv[ 1 ]; + } + ch->paired_idx = (int)ADMITTED_IDX; + ch->peer_hello.boot_id = boot_id; + ch->peer_hello.holds = (uchar)peer_holds; + fd_keepalive_init( ch->keepalive, ch->rng, FD_FAILOVER_CHANNEL_KEEPALIVE_NANOS, FD_FAILOVER_CHANNEL_IDLE_NANOS, now ); + fd_keepalive_rx( ch->keepalive, now ); + ch->paired_at = now; + sync_session( ctx ); + fd_memset( ch->peer_hello.junk_pubkey, 0x22, 32UL ); +} + +static void +make_history( fd_failover_history_t * history, + ulong tip ) { + ag_hist_t hist = { .anchor=tip, .last_leader_slot=FD_FAILOVER_SLOT_NULL, .vote_bound=FD_FAILOVER_SLOT_NULL, .rec_cnt=1UL }; + hist.rec[ 0 ].slot = tip; + hist.rec[ 0 ].flags = AG_HIST_FLAG_VOTED; + FD_TEST( !ag_hist_ser( &hist, history->state, sizeof(history->state), &history->sz ) ); + history->valid = 1; + history->tip = tip; +} + +/* A serialized history of rec_cnt notar votes ending at tip, as the + machine running the identity hands it over. */ +static void +make_peer_history( fd_failover_history_t * history, + ulong anchor, + ulong tip, + ulong rec_cnt ) { + static ag_hist_t hist; + fd_memset( &hist, 0, sizeof(hist) ); + hist.anchor = anchor; + hist.last_leader_slot = tip-3UL; + hist.rec_cnt = rec_cnt; + for( ulong i=0UL; istate, sizeof(history->state), &history->sz ) ); + history->valid = 1; + history->tip = tip; +} + +static void +deliver( fd_failover_tile_ctx_t * ctx, + ushort type, + void const * payload, + ulong payload_sz ) { + fd_memcpy( ctx->rx, payload, payload_sz ); + handle_control( ctx, type, payload_sz, 1000L ); +} + +static void +deliver_ack( fd_failover_tile_ctx_t * ctx, + ulong handoff_id ) { + uchar payload[ sizeof(fd_failover_promote_ack_t) ]; + deliver( ctx, (ushort)FD_FAILOVER_MSG_PROMOTE_ACK, payload, fd_failover_promote_ack_encode( payload, handoff_id ) ); +} + +static void +switch_ok( fd_failover_tile_ctx_t * ctx, + ulong watermark ) { + ctx->id_switch.result.result = FD_FAILOVER_SWITCH_OK; + ctx->id_switch.result.history_watermark = watermark; + FD_TEST( on_switch_resp_msg( ctx, ctx->id_switch.request_id ) ); +} + +static fd_failover_demoted_t +pending_demoted( fd_failover_tile_ctx_t const * ctx ) { + FD_TEST( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ); + fd_failover_demoted_t demoted; + FD_TEST( fd_failover_demoted_decode( &demoted, ctx->tx.payload, ctx->tx.sz ) ); + return demoted; +} + +static void +test_handoff( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); + make_history( &ctx->current_history, 99UL ); + fd_failover_handoff_request_t req = { .handoff_id=42UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH ); + step_controller( ctx, stem ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK && !ctx->tx.valid ); + switch_ok( ctx, 0UL ); + step_controller( ctx, stem ); + FD_TEST( ctx->holds==0 && fd_memeq( ctx->installed, ctx->hello.junk_pubkey, 32UL ) ); + fd_failover_demoted_t demoted = pending_demoted( ctx ); + FD_TEST( demoted.handoff_id==42UL && demoted.mode==FD_FAILOVER_MODE_ALPENGLOW && demoted.last_vote_slot==99UL ); + fd_adminctl_failover_status_resp_t status; + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.version==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION && !status.holds ); + FD_TEST( fd_memeq( status.target, ctx->hello.staked_pubkey, 32UL ) && fd_memeq( status.installed, ctx->hello.junk_pubkey, 32UL ) && + fd_memeq( status.junk, ctx->hello.junk_pubkey, 32UL ) ); + ctx->tx.valid = 0; + ctx->handoff.delivery = FD_FAILOVER_DELIVERY_SENT; + deliver_ack( ctx, 42UL ); + FD_TEST( ctx->taken && ctx->handoff.result==FD_FAILOVER_HANDOFF_TAKEN ); + controller_fini( ctx ); +} + +static void +test_no_final_state( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; + pair( ctx, 77UL, 0, 1000L ); + /* Before votor streams a history with a vote a request is refused. */ + fd_failover_handoff_request_t req = { .handoff_id=43UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=100UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + fd_failover_handoff_result_t result; + FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); + FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); + FD_TEST( result.handoff_id==43UL && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_IDLE ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.request_id && !ctx->promotion.adopt_id ); + char const * hint; + FD_TEST( !strcmp( control_refusal( result.result, &hint ), "NO_FINAL_HISTORY" ) ); + FD_TEST( strstr( hint, "the other machine" ) ); + /* The requester logs the relayed rejection under the same name. */ + FD_TEST( !strcmp( control_result( result.result, &hint ), "NO_FINAL_HISTORY" ) ); + ctx->tx.valid = 0; + ctx->session.close_after_send = 0; + /* A frame whose history is empty is not a final history. */ + static fd_votor_hist_msg_t hist; + fd_memset( &hist, 0, sizeof(hist) ); + fd_memcpy( hist.identity.uc, ctx->hello.staked_pubkey, 32UL ); + hist.replay_slot = 100UL; + hist.root_slot = 90UL; + hist.vote_slot = FD_FAILOVER_SLOT_NULL; + hist.hist.last_leader_slot = FD_FAILOVER_SLOT_NULL; + hist.hist.vote_bound = FD_FAILOVER_SLOT_NULL; + on_votor_hist_msg( ctx, &hist ); + FD_TEST( ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL && !ctx->current_history.valid ); + /* The first frame with a vote in it comes from a completed slot, not + a vote, and still becomes our final history. */ + hist.hist.anchor = 99UL; + hist.hist.rec_cnt = 1UL; + hist.hist.rec[ 0 ].slot = 99UL; + hist.hist.rec[ 0 ].flags = AG_HIST_FLAG_VOTED; + hist.vote_slot = 99UL; + on_votor_hist_msg( ctx, &hist ); + FD_TEST( ctx->last_vote_slot==99UL && ctx->own_floor==99UL && ctx->current_history.valid && ctx->current_history.tip==99UL ); + req.handoff_id = 44UL; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( !ctx->tx.valid && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->handoff.id==44UL ); + controller_fini( ctx ); +} + +/* test_request_replay_behind: a peer more than 64 slots behind our + last vote, or one with no replay slot yet, gets REPLAY_BEHIND and we + keep the identity, one exactly 64 behind is served. */ +static void +test_request_replay_behind( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); + ctx->last_vote_slot = 200UL; + make_history( &ctx->current_history, 200UL ); + ulong behind[ 2 ] = { 200UL-FD_FAILOVER_DEADLINE_SLOTS-1UL, FD_FAILOVER_SLOT_NULL }; + for( ulong i=0UL; i<2UL; i++ ) { + fd_failover_handoff_request_t req = { .handoff_id=49UL+i, .target_boot_id=OUR_BOOT_ID, .replay_slot=behind[ i ] }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + fd_failover_handoff_result_t result; + FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); + FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); + FD_TEST( result.handoff_id==49UL+i && result.result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->handoff.id ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.request_id ); + ctx->tx.valid = 0; + ctx->session.close_after_send = 0; + } + fd_failover_handoff_request_t req = { .handoff_id=51UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=200UL-FD_FAILOVER_DEADLINE_SLOTS }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->handoff.id==51UL ); + controller_fini( ctx ); +} + +/* test_demote_drain: after the junk switch the drain is its own action, + status reports it, and an ACK or the same request during it is ignored. */ +static void +test_demote_drain( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); + make_history( &ctx->current_history, 99UL ); + fd_failover_handoff_request_t req = { .handoff_id=44UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + step_controller( ctx, stem ); + ctx->hist_seen_seq = 796UL; + switch_ok( ctx, 800UL ); + step_controller( ctx, stem ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN ); + fd_adminctl_failover_status_resp_t status; + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.action==(uchar)FD_FAILOVER_ACTION_DEMOTE_DRAIN ); + deliver_ack( ctx, 44UL ); + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN && !ctx->taken && !ctx->tx.valid ); + ctx->hist_seen_seq = 799UL; + step_controller( ctx, stem ); + FD_TEST( ctx->holds==0 && pending_demoted( ctx ).handoff_id==44UL ); + controller_fini( ctx ); +} + +/* Deliver an OPERATOR frame from the admin tile. */ +static uchar admin_in_mem[ 1024 ] __attribute__((aligned(128))); + +static void +operator_frame( fd_failover_tile_ctx_t * ctx, + ulong epoch, + uchar key, + uchar failover_key ) { + ctx->admin_in_idx = 1UL; + ctx->admin_in_mem = (fd_wksp_t *)admin_in_mem; + ctx->admin_in_chunk0 = 0UL; + ctx->admin_in_wmark = 0UL; + fd_failover_bus_msg_t * msg = (fd_failover_bus_msg_t *)admin_in_mem; + fd_memset( msg, 0, sizeof(*msg) ); + fd_failover_operator_t operator = { .epoch=epoch }; + fd_memset( operator.identity, key, 32UL ); + fd_memset( operator.failover_identity, failover_key, 32UL ); + fd_memcpy( msg->payload, &operator, sizeof(operator) ); + FD_TEST( !before_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR_ACTION ) ); + during_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR_ACTION, 0UL, sizeof(*msg), 0UL ); + after_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR_ACTION, sizeof(*msg), 0UL, 0UL, stem ); +} + +/* test_operator_switch: set-identity during a demotion. We restart on + the new key with a new boot and nothing in flight, the answer to the + switch we asked for before is dropped, the next request has the new + failover epoch, and a request bound to our old boot cannot demote us. */ +static void +test_operator_switch( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); + make_history( &ctx->current_history, 99UL ); + ctx->own_floor = 99UL; + fd_failover_handoff_request_t req = { .handoff_id=47UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + step_controller( ctx, stem ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK ); + ulong nonce = ctx->id_switch.request_id; + + /* set-identity installs the junk key. */ + operator_frame( ctx, 1UL, 0x11, 0x5A ); + FD_TEST( ctx->holds==0 && ctx->hello.holds==0 && fd_memeq( ctx->installed, ctx->hello.junk_pubkey, 32UL ) ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->stuck && !ctx->tx.valid && !ctx->handoff.id && !ctx->request.id ); + FD_TEST( ctx->hello.boot_id!=OUR_BOOT_ID && ctx->channel->self_hello.boot_id==ctx->hello.boot_id ); + FD_TEST( !ctx->current_history.valid && ctx->own_floor==99UL ); + FD_TEST( ctx->id_switch.epoch==1UL && ctx->id_switch.discard && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK ); + FD_TEST( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ); + + /* The late answer ends the old switch and is dropped. */ + ctx->id_switch.result.result = FD_FAILOVER_SWITCH_OK; + FD_TEST( !on_switch_resp_msg( ctx, nonce ) ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.fresh && !ctx->id_switch.discard ); + + /* The same or an older frame changes nothing. */ + ulong boot_id = ctx->hello.boot_id; + operator_frame( ctx, 1UL, 0x5A, 0x5A ); + FD_TEST( ctx->hello.boot_id==boot_id && ctx->holds==0 ); + + /* The next switch request has the new failover epoch. */ + FD_TEST( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_STAKED )!=ULONG_MAX ); + fd_failover_switch_req_t sent; + fd_memcpy( &sent, ((fd_failover_bus_msg_t const *)bus_mem)->payload, sizeof(sent) ); + FD_TEST( sent.epoch==1UL ); + + /* The request crossed another set-identity, with the staked key, whose + OPERATOR frame has not arrived, and the STALE refusal catches us up. */ + ctx->admin_in_idx = 1UL; + ctx->id_switch.response.result = FD_FAILOVER_SWITCH_ERR_STALE; + ctx->id_switch.response.operator.epoch = 2UL; + fd_memset( ctx->id_switch.response.operator.identity, 0x5A, 32UL ); + fd_memset( ctx->id_switch.response.operator.failover_identity, 0x5A, 32UL ); + ctx->id_switch.response_nonce = ctx->id_switch.request_id; + after_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_SWITCH_RESP, sizeof(fd_failover_bus_msg_t), 0UL, 0UL, stem ); + FD_TEST( ctx->holds==1 && ctx->id_switch.epoch==2UL && ctx->hello.boot_id!=boot_id && fd_memeq( ctx->installed, ctx->hello.staked_pubkey, 32UL ) ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.fresh && ctx->action==FD_FAILOVER_ACTION_IDLE ); + + /* A request bound to our first boot is refused, one to this boot is + served once we have a final history. */ + pair( ctx, 78UL, 0, 1000L ); + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE ); + ctx->tx.valid = 0; + ctx->session.close_after_send = 0; + ctx->last_vote_slot = 120UL; + make_history( &ctx->current_history, 120UL ); + fd_failover_handoff_request_t fresh = { .handoff_id=48UL, .target_boot_id=ctx->hello.boot_id, .replay_slot=120UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &fresh, sizeof(fresh) ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->handoff.id==48UL ); + controller_fini( ctx ); +} + +/* test_operator_failover_identity: HELLO has the failover identity + OPERATOR gives, and we run it when the installed key is that + identity. */ +static void +test_operator_failover_identity( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); + ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); + ctx->staked_seen_at = 900L; + operator_frame( ctx, 1UL, 0x77, 0x77 ); + uchar third[ 32 ]; fd_memset( third, 0x77, 32UL ); + FD_TEST( ctx->holds==1 && ctx->hello.holds==1 && fd_memeq( ctx->installed, third, 32UL ) ); + FD_TEST( fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) && fd_memeq( ctx->channel->self_hello.staked_pubkey, third, 32UL ) ); + FD_TEST( !ctx->staked_addr && !ctx->staked_seen_at ); + + /* A junk set-identity keeps the failover identity, the configured key does not come back. */ + operator_frame( ctx, 2UL, 0x11, 0x77 ); + FD_TEST( ctx->holds==0 && fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) ); + + /* Another installed key, as after failover promote prepared + the identity, means we do not run it. */ + operator_frame( ctx, 3UL, 0x5A, 0x77 ); + uchar other[ 32 ]; fd_memset( other, 0x5A, 32UL ); + FD_TEST( ctx->holds==0 && fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) && fd_memeq( ctx->installed, other, 32UL ) ); + + /* A new target must be reflected in both admission and HELLO. */ + operator_frame( ctx, 4UL, 0x11, 0x77 ); + uchar public[ 32 ]; fd_memset( public, 0x5A, 32UL ); + fd_failover_hello_t peer = ctx->channel->self_hello; + peer.holds = 1; + peer.boot_id = 77UL; + fd_memset( peer.junk_pubkey, 0x22, 32UL ); + FD_TEST( fd_failover_hello_check( &ctx->channel->self_hello, &peer )==FD_FAILOVER_HELLO_OK ); + fd_memcpy( peer.staked_pubkey, public, 32UL ); + FD_TEST( fd_failover_hello_check( &ctx->channel->self_hello, &peer )==FD_FAILOVER_HELLO_ERR_STAKED ); + controller_fini( ctx ); +} + +/* test_standby_vote_counts: a vote our history published counts toward + the floor whatever our role, set-identity can install the staked key + before we hear of it. */ +static void +test_standby_vote_counts( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); + fd_votor_hist_msg_t hist; + fd_memset( &hist, 0, sizeof(hist) ); + fd_memcpy( hist.identity.uc, ctx->hello.staked_pubkey, 32UL ); + hist.replay_slot = 130UL; + hist.root_slot = FD_FAILOVER_SLOT_NULL; + hist.has_vote = 1; + hist.vote_slot = 129UL; + on_votor_hist_msg( ctx, &hist ); + FD_TEST( ctx->own_floor==129UL && ctx->last_vote_slot==129UL ); + controller_fini( ctx ); +} + +/* A request of ours bound to boot 77 of the member whose junk key was + 0x22, as the tests' pair() makes it. A new boot generates a new junk + key, so the restart tests pair with another one. */ +static void +open_request( fd_failover_tile_ctx_t * ctx, + ulong action ) { + ctx->request.id = 900UL; + ctx->request.last_id = 900UL; + ctx->request.result = FD_FAILOVER_HANDOFF_PENDING; + ctx->request.addr = FD_IP4_ADDR(10,0,0,2); + ctx->request.until = fd_long_sat_add( 1000L, FD_FAILOVER_CHANNEL_IDLE_NANOS ); + ctx->request.peer.boot_id = 77UL; + fd_memset( ctx->request.peer.junk, 0x22, 32UL ); + ctx->last_requested = 1; + ctx->action = action; +} + +/* test_result_wait_peer_restart: the peer that ran the identity + restarts before it confirms our ACK. Its new boot cannot, so the wait + ends and we keep the identity. A request waiting to have its refusal + confirmed ends the same way, without the identity. A request paused + at its deadline ends the same way when the new boot pairs, so the old + holder's next request is not refused as in progress forever. */ +static void +test_result_wait_peer_restart( void ) { + for( int paused=0; paused<2; paused++ ) { + for( int holds=0; holds<2; holds++ ) { + fd_failover_tile_ctx_t * ctx = controller_init( holds ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + if( paused ) ctx->request.until = 0L; + pair( ctx, 78UL, 0, 1000L ); + fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x33, 32UL ); + int busy = 0; + peer_poll( ctx, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->request.id && ctx->request.last_id==900UL ); + FD_TEST( ctx->request.result==( holds ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ) ); + FD_TEST( ctx->holds==holds ); + FD_TEST( ctx->stuck==!holds ); + controller_fini( ctx ); + } + } + + /* A key switch in flight keeps the paused wait. */ + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + ctx->request.until = 0L; + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_STAKED; + pair( ctx, 78UL, 0, 1000L ); + fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x33, 32UL ); + int busy = 0; + peer_poll( ctx, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && ctx->request.id==900UL && !ctx->request.until ); + controller_fini( ctx ); + + /* A paused request for final state keeps waiting for the operator. */ + ctx = controller_init( 0 ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ); + ctx->request.until = 0L; + pair( ctx, 78UL, 1, 1000L ); + fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x33, 32UL ); + busy = 0; + peer_poll( ctx, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && ctx->request.id==900UL && !ctx->request.until && !ctx->request.peer_restarted ); + controller_fini( ctx ); + + /* While we wait for final state a restart still pauses the request. */ + ctx = controller_init( 0 ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ); + pair( ctx, 78UL, 1, 1000L ); + fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x33, 32UL ); + busy = 0; + peer_poll( ctx, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && ctx->request.id && !ctx->request.until && ctx->stuck ); + controller_fini( ctx ); + + /* `failover promote` that only ends a paused confirmation wait answers + with the id of the handoff it ended, not 0. */ + ctx = controller_init( 1 ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + ctx->request.until = 0L; + ctx->request.peer_restarted = 1; + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_HANDOFF }; + ctx->bus_req.nonce = 8UL; + fd_memcpy( ctx->bus_req.payload, &req, sizeof(req) ); + serve_bus_request( ctx, stem, 1000L ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->request.id && ctx->request.last_id==900UL && ctx->request.result==FD_FAILOVER_HANDOFF_TAKEN ); + fd_failover_bus_msg_t const * out = (fd_failover_bus_msg_t const *)bus_mem; + fd_adminctl_failover_control_resp_t response; + fd_memcpy( &response, out->payload, sizeof(response) ); + FD_TEST( out->nonce==8UL && out->result==FD_ADMINCTL_RESULT_SUCCESS && response.handoff_id==900UL && response.holds ); + controller_fini( ctx ); +} + +/* test_session_change_drops_result: a queued HANDOFF_RESULT belongs to + its session, a new one drops it and the requester asks again. A + queued DEMOTED stays for the handoff target. */ +static void +test_session_change_drops_result( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); + fd_failover_handoff_request_t req = { .handoff_id=46UL, .target_boot_id=OUR_BOOT_ID+1UL, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT && ctx->session.close_after_send ); + ctx->channel->generation++; + pair( ctx, 79UL, 0, 1000L ); + FD_TEST( !ctx->tx.valid && !ctx->session.close_after_send ); + controller_fini( ctx ); + + ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); + make_history( &ctx->current_history, 99UL ); + req.target_boot_id = OUR_BOOT_ID; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + step_controller( ctx, stem ); + switch_ok( ctx, 0UL ); + step_controller( ctx, stem ); + FD_TEST( pending_demoted( ctx ).handoff_id==46UL ); + ctx->channel->generation++; + pair( ctx, 80UL, 0, 1000L ); + FD_TEST( pending_demoted( ctx ).handoff_id==46UL ); + controller_fini( ctx ); +} + +static ulong +operator_cmd( fd_failover_tile_ctx_t * ctx, + ulong cmd, + uint addr, + ushort port ) { + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=cmd, .addr=addr, .port=port }; + return apply_control( ctx, &req, 1000L ); +} + +/* test_request_address: status shows where the open request dials, a + gossip address fixed when it opened, and after it the live one. A + resume replaces the address or port it gives and keeps the other, one + that gives neither keeps both, and a bound request takes a given + address too. */ +static void +test_request_address( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); + ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + ctx->staked_addr = FD_IP4_ADDR(10,0,0,3); + fd_adminctl_failover_status_resp_t status; + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.peer_addr==FD_IP4_ADDR(10,0,0,2) && !status.peer_addr_cmd && status.peer_port==8010 ); + + request_pause( ctx, 1000L, "test" ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, FD_IP4_ADDR(10,0,0,4), 9001 )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,4) && dial_port( ctx )==9001 ); + request_pause( ctx, 1000L, "test" ); + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.request_paused && status.mode==(uchar)FD_FAILOVER_MODE_ALPENGLOW ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, FD_IP4_ADDR(10,0,0,5), 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,5) && dial_port( ctx )==9001 ); + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.peer_addr==FD_IP4_ADDR(10,0,0,5) && status.peer_addr_cmd && status.peer_port==9001 && !status.request_paused ); + request_pause( ctx, 1000L, "test" ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 9002 )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,5) && dial_port( ctx )==9002 ); + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.peer_addr_cmd ); + request_pause( ctx, 1000L, "test" ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,5) && dial_port( ctx )==9002 ); + ctx->request.peer.boot_id = 77UL; + request_pause( ctx, 1000L, "test" ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, FD_IP4_ADDR(10,0,0,6), 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,6) && ctx->request.peer.boot_id==77UL ); + + request_end( ctx, 1000L, FD_FAILOVER_HANDOFF_DECLINED ); + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.peer_addr==FD_IP4_ADDR(10,0,0,3) && !status.peer_addr_cmd && status.peer_port==8010 ); + controller_fini( ctx ); +} + +/* test_abort_answers_requester: a handoff demotion whose junk switch + fails keeps the identity and tells the requester at once. */ +static void +test_abort_answers_requester( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); + make_history( &ctx->current_history, 99UL ); + fd_failover_handoff_request_t req = { .handoff_id=47UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + step_controller( ctx, stem ); + ctx->id_switch.result.result = FD_FAILOVER_SWITCH_ERR_DISABLED; + FD_TEST( on_switch_resp_msg( ctx, ctx->id_switch.request_id ) ); + step_controller( ctx, stem ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_IDLE && ctx->stuck ); + fd_failover_handoff_result_t result; + FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT && ctx->tx.to.boot_id==77UL && ctx->session.close_after_send ); + FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); + FD_TEST( result.handoff_id==47UL && result.result==FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + controller_fini( ctx ); +} + +/* test_wait_result_redials: a request that paused while its promotion + ran dials again once the promotion ends, so the bound peer can confirm + and a restarted one ends the request. */ +static void +test_wait_result_redials( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); + open_request( ctx, FD_FAILOVER_ACTION_PROMOTE_SWITCH ); + ctx->request.until = 0L; + ctx->promotion.from_peer = 1; + ctx->promotion.peer = ctx->request.peer; + ctx->promotion.handoff_id = 900UL; + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_STAKED; + ctx->id_switch.request_id = 1UL; + switch_ok( ctx, 0UL ); + step_controller( ctx, stem ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && fd_memeq( ctx->installed, ctx->hello.staked_pubkey, 32UL ) ); + FD_TEST( ctx->request.until && ctx->channel->peer_addr==FD_IP4_ADDR(10,0,0,2) ); + controller_fini( ctx ); +} + +/* test_refused_command_keeps_stuck: only an accepted command clears + stuck, and the takeover checks behind a refusal report what they + find. */ +static void +test_refused_command_keeps_stuck( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); + ctx->stuck = 1; + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS && ctx->stuck ); + ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS && !ctx->stuck ); + request_end( ctx, 1000L, FD_FAILOVER_HANDOFF_DECLINED ); + controller_fini( ctx ); + + /* Without --force the takeover checks report SUCCESS when none finds + another machine running the identity, and the refusal otherwise. */ + ctx = controller_init( 0 ); + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + fd_adminctl_failover_status_resp_t status; + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.promote_result==FD_ADMINCTL_RESULT_SUCCESS && !status.holds ); + FD_TEST( fd_memeq( status.installed, ctx->hello.junk_pubkey, 32UL ) && fd_memeq( status.target, ctx->hello.staked_pubkey, 32UL ) ); + ctx->taken = 1; + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_FAILOVER_CONTROL_RESULT_TAKEN ); + FD_TEST( promote_guard( ctx, 1000L, 1 )==FD_ADMINCTL_RESULT_SUCCESS ); + ctx->taken = 0; + ctx->staked_seen_at = 1000L; + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN ); + FD_TEST( promote_guard( ctx, 1000L+FD_FAILOVER_GOSSIP_FRESH_NANOS, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + ctx->staked_seen_at = 0L; + pair( ctx, 77UL, 1, 1000L ); + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS ); + ctx->channel->peer_hello.holds = 0; + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + ctx->holds = 1; + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER ); + controller_fini( ctx ); +} + +/* test_dialed_non_holder: a request that pairs with a peer that does not + run the identity ends as its own outcome, not as declined. */ +static void +test_dialed_non_holder( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); + ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + pair( ctx, 77UL, 0, 1000L ); + int busy = 0; + peer_poll( ctx, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->request.id && ctx->request.result==FD_FAILOVER_HANDOFF_NOT_HOLDER ); + controller_fini( ctx ); +} + +/* test_restarted_peer_new_request: once the peer boot a request is + bound to restarted, `failover promote` starts a new request instead of + resuming one that can never finish, and the accepted command answers + with its id. */ +static void +test_restarted_peer_new_request( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); + ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ); + pair( ctx, 78UL, 1, 1000L ); + int busy = 0; + peer_poll( ctx, 1000L, &busy ); + FD_TEST( ctx->request.id==900UL && ctx->request.peer_restarted && !ctx->request.until ); + + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_HANDOFF }; + ctx->bus_req.nonce = 7UL; + fd_memcpy( ctx->bus_req.payload, &req, sizeof(req) ); + serve_bus_request( ctx, stem, 1000L ); + FD_TEST( ctx->request.id && ctx->request.id!=900UL && !ctx->request.peer.boot_id && !ctx->request.peer_restarted ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ); + fd_failover_bus_msg_t const * out = (fd_failover_bus_msg_t const *)bus_mem; + fd_adminctl_failover_control_resp_t response; + fd_memcpy( &response, out->payload, sizeof(response) ); + FD_TEST( out->nonce==7UL && out->result==FD_ADMINCTL_RESULT_SUCCESS && response.handoff_id==ctx->request.id ); + controller_fini( ctx ); +} + +/* test_force_empty_bound: a forced promotion with no saved history sets + its vote bound at the replay slot or the coverage floor, whichever is + higher. */ +static void +test_force_empty_bound( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); + fd_cstr_ncpy( ctx->promotion.label, "promotion 1", sizeof(ctx->promotion.label) ); + ctx->promotion.source = FD_FAILOVER_SOURCE_EMPTY; + ctx->promotion.force = 1; + ulong const floors[ 3 ] = { FD_FAILOVER_SLOT_NULL, 120UL, 80UL }; + ulong const fences[ 3 ] = { 100UL, 120UL, 100UL }; + for( ulong i=0UL; i<3UL; i++ ) { + ctx->promotion.floor = floors[ i ]; + promote_start( ctx ); + FD_TEST( replay_ready( ctx ) ); + FD_TEST( ctx->empty_vote_after==fences[ i ] ); + FD_TEST( ctx->promotion.adopt.sz==FD_VOTOR_ADOPT_EMPTY_SZ && FD_LOAD( ulong, ctx->promotion.adopt.state )==fences[ i ] ); + } + controller_fini( ctx ); +} + +static void +test_adopt_capacity_retry( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); + make_peer_history( &ctx->peer_history, 90UL, 99UL, 6UL ); + fd_failover_peer_t peer = { .boot_id = 77UL }; + start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &peer, 42UL, 0 ); + step_promote_wait_replay( ctx, stem ); + ulong id = ctx->promotion.adopt_id; + fd_failover_history_t saved = ctx->promotion.adopt; + ctx->adopt_in_idx = 0UL; + ctx->adopt_result.result = FD_VOTOR_ADOPT_ERR_CAPACITY; + after_frag( ctx, 0UL, 0UL, id, sizeof(ctx->adopt_result), 0UL, 0UL, stem ); + step_promote_wait_adopt( ctx, stem ); + FD_TEST( ctx->promotion.retry && ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ); + FD_TEST( ctx->holds==0 && !ctx->stuck && !ctx->tx.valid ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ); + FD_TEST( ctx->peer_history.valid && fd_memeq( &saved, &ctx->promotion.adopt, sizeof(saved) ) ); + step_promote_wait_adopt( ctx, stem ); + FD_TEST( ctx->promotion.adopt_id==id ); + + ctx->replay_slot++; + step_promote_wait_adopt( ctx, stem ); + FD_TEST( !ctx->promotion.retry && ctx->promotion.adopt_id!=id ); + FD_TEST( fd_memeq( bus_mem, ctx->hello.staked_pubkey, FD_VOTOR_ADOPT_REQ_HDR_SZ ) && fd_memeq( bus_mem+FD_VOTOR_ADOPT_REQ_HDR_SZ, saved.state, saved.sz ) ); + ctx->adopt_result = (fd_votor_adopt_result_t){ .result=FD_VOTOR_ADOPT_SUCCESS, .root=90UL, .vote_slot=99UL, .vote_bound=ULONG_MAX }; + after_frag( ctx, 0UL, 1UL, ctx->promotion.adopt_id, sizeof(ctx->adopt_result), 0UL, 0UL, stem ); + step_promote_wait_adopt( ctx, stem ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH ); + controller_fini( ctx ); +} + +/* A full pool refuses new candidates. The outbound and admitted slots + are never handed to an unknown address, the known address gets the + admitted slot and nothing else. */ +static void +fill( fd_failover_channel_t * ch, + ulong idx, + uint address, + long deadline ) { + struct candidate * c = &ch->candidates[ idx ]; + if( c->fd!=-1 ) close( c->fd ); + fd_memset( c, 0, sizeof(*c) ); + c->fd = open( "/dev/null", O_RDONLY|O_CLOEXEC ); + FD_TEST( c->fd!=-1 ); + c->tls.fd = -1; + c->phase = PHASE_TLS; + c->address = address; + c->deadline = deadline; +} + +static void +test_listen_slot( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + fd_failover_channel_t * ch = ctx->channel; + long now = FD_FAILOVER_ADMISSION_START_NANOS; + uint stranger = FD_IP4_ADDR(10,2,0,1); + uint known = FD_IP4_ADDR(10,2,0,2); + remember_source( ch, known, now ); + FD_TEST( listen_slot( ch, stranger, now )==UNAUTHENTICATED_IDX ); + FD_TEST( listen_slot( ch, known, now )==ADMITTED_IDX ); + for( ulong i=0UL; icandidates[ UNAUTHENTICATED_IDX ].fd ); + ch->candidates[ UNAUTHENTICATED_IDX ].fd = -1; + FD_TEST( listen_slot( ch, stranger, now )==UNAUTHENTICATED_IDX ); + FD_TEST( listen_slot( ch, known, now )==FD_FAILOVER_CHANNEL_CANDIDATE_MAX ); /* only the admitted slot, never the pool */ + FD_TEST( ch->candidates[ OUTBOUND_IDX ].fd!=-1 && ch->candidates[ ADMITTED_IDX ].fd!=-1 ); + FD_TEST( listen_slot( ch, known, now+FD_FAILOVER_CHANNEL_IDLE_NANOS )==UNAUTHENTICATED_IDX ); /* forgotten once idle */ + for( ulong i=0UL; icandidates[ i ].fd!=-1 ) { + close( ch->candidates[ i ].fd ); + ch->candidates[ i ].fd = -1; + } + controller_fini( ctx ); +} + +int +main( int argc, char ** argv ) { + fd_boot( &argc, &argv ); + FD_TEST( fd_failover_channel_footprint()<=sizeof(ch_mem) ); + test_handoff(); + test_no_final_state(); + test_request_replay_behind(); + test_demote_drain(); + test_operator_switch(); + test_operator_failover_identity(); + test_standby_vote_counts(); + test_result_wait_peer_restart(); + test_session_change_drops_result(); + test_request_address(); + test_abort_answers_requester(); + test_wait_result_redials(); + test_refused_command_keeps_stuck(); + test_dialed_non_holder(); + test_restarted_peer_new_request(); + test_force_empty_bound(); + test_adopt_capacity_retry(); + test_listen_slot(); + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} diff --git a/src/discof/failover/test_failover_wire.c b/src/discof/failover/test_failover_wire.c new file mode 100644 index 00000000000..58b773cefee --- /dev/null +++ b/src/discof/failover/test_failover_wire.c @@ -0,0 +1,44 @@ +#include "fd_failover_wire.h" +#include "../../util/fd_util.h" + +static uchar frame[ FD_FAILOVER_FRAME_MAX ]; +static uchar big[ 2UL*FD_FAILOVER_FRAME_MAX ]; + +static void +test_roundtrip( void ) { + + uchar payload[ 70 ]; + for( ulong i=0UL; i<70UL; i++ ) payload[ i ] = (uchar)i; + + ulong sz = fd_failover_wire_encode( frame, (ushort)FD_FAILOVER_MSG_DEMOTED, payload, 70UL ); + FD_TEST( sz==FD_FAILOVER_FRAME_HDR_SZ+70UL ); + + ushort type; + uchar const * out; + ulong out_sz; + ulong frame_sz; + FD_TEST( fd_failover_wire_decode( frame, sz, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); + FD_TEST( type==(ushort)FD_FAILOVER_MSG_DEMOTED ); + FD_TEST( out_sz==70UL ); + FD_TEST( frame_sz==sz ); + FD_TEST( fd_memeq( out, payload, 70UL ) ); + + ulong sz2 = fd_failover_wire_encode( big, (ushort)FD_FAILOVER_MSG_HELLO, payload, 10UL ); + fd_memset( big+sz2, 0x77, 64UL ); + FD_TEST( fd_failover_wire_decode( big, sz2+64UL, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); + FD_TEST( frame_sz==sz2 ); + + FD_LOG_NOTICE(( "pass: test_roundtrip" )); +} + +int +main( int argc, + char ** argv ) { + fd_boot( &argc, &argv ); + + test_roundtrip(); + + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} diff --git a/src/discof/gossip/fd_gossvf_tile.c b/src/discof/gossip/fd_gossvf_tile.c index ad75b7136db..8a57ecb7a5f 100644 --- a/src/discof/gossip/fd_gossvf_tile.c +++ b/src/discof/gossip/fd_gossvf_tile.c @@ -618,14 +618,20 @@ check_duplicate_instance( fd_gossvf_tile_ctx_t * ctx, FD_LOG_CRIT(( "unexpected message tag %u", view->tag )); } + long our_creation_nanos = ctx->instance_creation_wallclock_nanos; for( ulong i=0UL; itag!=FD_GOSSIP_VALUE_CONTACT_INFO ) ) continue; - if( FD_LIKELY( ctx->instance_creation_wallclock_nanos>=FD_MICRO_TO_NANOSEC( value->contact_info->outset ) ) ) continue; - if( FD_LIKELY( memcmp( ctx->identity_pubkey->uc, value->origin, 32UL ) ) ) continue; - - FD_LOG_ERR(( "duplicate running instances of the same validator node, our timestamp: %ldns their timestamp: %ldns", ctx->instance_creation_wallclock_nanos, FD_MICRO_TO_NANOSEC( value->contact_info->outset ) )); + long their_creation_nanos = FD_MICRO_TO_NANOSEC( value->contact_info->outset ); + if( FD_UNLIKELY( their_creation_nanos>our_creation_nanos && + !memcmp( ctx->identity_pubkey->uc, value->origin, 32UL ) ) ) { + FD_LOG_ERR(( "duplicate running instances of the same validator node, our timestamp: %ldns their timestamp: %ldns. " + "If the newer instance is a failover machine that restarted with this identity in [paths.identity_key], stop it, " + "enable [failover.boot_to_standby] and/or set its [paths.identity_key] to a random junk key, restart it, " + "then check `failover status` on both machines.", + our_creation_nanos, their_creation_nanos )); + } } } diff --git a/src/discof/votor/fd_votor_tile.c b/src/discof/votor/fd_votor_tile.c index 3474f67cf84..10569f203f7 100644 --- a/src/discof/votor/fd_votor_tile.c +++ b/src/discof/votor/fd_votor_tile.c @@ -47,6 +47,7 @@ #define IN_KIND_NET (3) #define IN_KIND_REPLAY (4) #define IN_KIND_SIGN (5) +#define IN_KIND_FAILOV (6) #define OUT_IDX_VOTOR (0UL) #define OUT_IDX_NET (1UL) @@ -391,6 +392,39 @@ struct fd_votor_tile { ulong leader_window_superseded; ulong cert_tx_unstaked; } metrics; + + /* Failover, on when the failover links are in the topology */ + + int failover_enabled; /* the topology has the failover links */ + int installed_by_failover; /* the identity we run came from a failover switch, not boot or set-identity */ + fd_pubkey_t junk_key; /* the junk key generated at boot, a handoff switches to it */ + ulong highest_unsent_slot; /* highest vote the tile built but did not send, ULONG_MAX none */ + int failover_hist_adopted; /* a history, possibly empty, was adopted since the last switch */ + ulong adopted_tip; /* tip of the adopted history, ULONG_MAX for an empty one or none */ + int vote_authority; /* our rank may be installed, 0 keeps us unranked */ + int hist_was_truncated; /* the last export dropped windows, warned once */ + ulong last_leader_slot; /* slot of the last LEADER we published, ULONG_MAX before any */ + ulong adopted_last_leader_slot; /* window of the adopted last LEADER or vote bound, ULONG_MAX when none */ + ulong last_vote_slot; /* highest slot we broadcast a vote for, ULONG_MAX before any */ + fd_pubkey_t last_vote_key; /* the identity that vote went out as */ + ulong root_slot; /* replay's root at the last completed slot, ULONG_MAX before any */ + ulong highest_completed_slot; /* highest slot replay completed, 0 before any */ + + ulong hist_out_idx; /* votor_hist, ULONG_MAX without failover */ + void * hist_out_mem; + ulong hist_out_chunk0; + ulong hist_out_wmark; + ulong hist_out_chunk; + ulong hist_seq; /* votor_hist sequence after our last frame */ + + ulong failov_out_idx; /* votor_failov, our adoption responses */ + void * failov_out_mem; + ulong failov_out_chunk0; + ulong failov_out_wmark; + ulong failov_out_chunk; + + uchar adopt_req[ FD_VOTOR_ADOPT_REQ_HDR_SZ+AG_HIST_SER_MAX ]; + ulong adopt_req_sz; }; typedef struct fd_votor_tile fd_votor_tile_t; @@ -1143,6 +1177,7 @@ update_own_rank( fd_votor_tile_t * ctx ) { for( ulong rank=0UL; rankvalidator_cnt; rank++ ) { if( FD_UNLIKELY( !memcmp( epoch_info->validators[ rank ].id_key, ctx->id_key.uc, sizeof(ag_id_key_t) ) ) ) own_rank = (ushort)rank; } + if( FD_UNLIKELY( ctx->failover_enabled && !ctx->vote_authority ) ) own_rank = USHORT_MAX; ag_pool_set_rank ( ctx->pool, epoch_slots[ i ], own_rank ); ag_votor_set_rank( ctx->votor, epoch_slots[ i ], own_rank ); } @@ -1159,6 +1194,7 @@ update_own_bls_key( fd_votor_tile_t * ctx ) { for( ulong rank=0UL; rankvalidator_cnt; rank++ ) { if( FD_UNLIKELY( !memcmp( epoch_info->validators[ rank ].id_key, ctx->id_key.uc, sizeof(ag_id_key_t) ) ) ) own_rank = (ushort)rank; } + if( FD_UNLIKELY( ctx->failover_enabled && !ctx->vote_authority ) ) own_rank = USHORT_MAX; /* with failover we stay unranked while we may not vote */ uchar const * own_bls_key = NULL; /* NULL if we are not ranked or cannot sign with our key, in which case we do not vote */ if( FD_LIKELY( own_rank!=USHORT_MAX ) ) { ag_validator_info_t const * validator = &epoch_info->validators[ own_rank ]; @@ -1329,6 +1365,323 @@ rank_voters( ag_epoch_info_t * epoch_info, return epoch_info; } +/* Returns the highest slot whose window this identity may have led, + ULONG_MAX when there is none. */ +static ulong +leader_floor( fd_votor_tile_t const * ctx ) { + if( ctx->last_leader_slot==ULONG_MAX ) return ctx->adopted_last_leader_slot; + if( ctx->adopted_last_leader_slot==ULONG_MAX ) return ctx->last_leader_slot; + return fd_ulong_max( ctx->last_leader_slot, ctx->adopted_last_leader_slot ); +} + +/* The end of the highest window in which our slot states may record a + vote that did not go out: under the restart floor, under the catch-up + floor, or dropped by this tile. ULONG_MAX when there is none. */ +static ulong +failover_unsent_bound( fd_votor_tile_t const * ctx ) { + ulong slots[ 3 ] = { ctx->vote_history_slot ? ctx->vote_history_slot-1UL : ULONG_MAX, + ctx->catchup_vote_slot ? ctx->catchup_vote_slot-1UL : ULONG_MAX, + ctx->highest_unsent_slot }; + ulong bound = ULONG_MAX; + for( ulong i=0UL; i<3UL; i++ ) { + if( slots[ i ]==ULONG_MAX ) continue; + ulong end = ag_first_slot_in_window( slots[ i ] )+AG_SLOTS_PER_WINDOW-1UL; + bound = bound==ULONG_MAX ? end : fd_ulong_max( bound, end ); + } + return bound; +} + +/* Publishes a votor_hist frame to the failover tile, see + fd_votor_hist_msg. The export reads consensus state and changes none + of it, so a validator that never adopts a history votes as one + without failover. */ +static void +publish_hist( fd_votor_tile_t * ctx, + fd_stem_context_t * stem, + int has_vote ) { + if( FD_UNLIKELY( ctx->hist_out_idx==ULONG_MAX ) ) return; + fd_votor_hist_msg_t * msg = fd_chunk_to_laddr( ctx->hist_out_mem, ctx->hist_out_chunk ); + msg->identity = ctx->id_key; + msg->replay_slot = ctx->highest_completed_slot ? ctx->highest_completed_slot : ULONG_MAX; + msg->root_slot = ctx->root_slot; + msg->has_vote = has_vote; + msg->truncated = ag_votor_hist_export( ctx->votor, leader_floor( ctx ), &msg->hist ); + if( FD_UNLIKELY( msg->truncated && !ctx->hist_was_truncated ) ) FD_LOG_WARNING(( "vote history export dropped windows below slot %lu, finality is far behind our votes", ag_hist_first_slot( msg->hist.anchor ) )); + ctx->hist_was_truncated = msg->truncated; + /* The floors this tile filters votes under, and the votes it built but + did not send, travel to the adopter as the history's vote bound. */ + ulong unsent = failover_unsent_bound( ctx ); + if( FD_UNLIKELY( unsent!=ULONG_MAX && ( msg->hist.vote_bound==ULONG_MAX || msg->hist.vote_boundhist.vote_bound = unsent; + msg->vote_slot = ag_hist_tip( &msg->hist ); /* the failover tile checks the history ends here */ + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->hist_out_idx, FD_VOTOR_HIST_SIG, ctx->hist_out_chunk, sizeof(fd_votor_hist_msg_t), 0UL, tspub, tspub ); + ctx->hist_out_chunk = fd_dcache_compact_next( ctx->hist_out_chunk, sizeof(fd_votor_hist_msg_t), ctx->hist_out_chunk0, ctx->hist_out_wmark ); + ctx->hist_seq = stem->seqs[ ctx->hist_out_idx ]; +} + +/* Raises the adopted leader floor to the window of slot, so we do not + lead that window or one below it. */ +static void +raise_leader_floor( fd_votor_tile_t * ctx, + ulong slot ) { + if( FD_UNLIKELY( slot==ULONG_MAX ) ) return; + ulong window = ag_first_slot_in_window( slot ); + if( FD_LIKELY( ctx->adopted_last_leader_slot==ULONG_MAX || ctx->adopted_last_leader_slotadopted_last_leader_slot = window; +} + +/* An empty request is shorter than any serialized history */ +FD_STATIC_ASSERT( FD_VOTOR_ADOPT_EMPTY_SZAG_HIST_SER_MAX || ag_hist_de( req, req_sz, hist ) ) ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history of %lu bytes that does not decode", req_sz )); + return result; + } + + result.result = FD_VOTOR_ADOPT_ERR_UNREPLAYED; + if( FD_UNLIKELY( !ctx->init ) ) { + FD_LOG_WARNING(( "failover: not adopting the vote history, consensus is not up yet" )); + return result; + } + result.result = FD_VOTOR_ADOPT_ERR_INVALID; + + /* An empty request sets the vote bound and leader floor to the slot it + gives. */ + if( FD_UNLIKELY( empty ) ) { + ulong bound = FD_LOAD( ulong, req ); + if( FD_UNLIKELY( bound==ULONG_MAX ) ) { + FD_LOG_WARNING(( "failover: refusing an empty vote history that has no bound slot" )); + return result; + } + ag_votor_set_vote_bound( ctx->votor, bound ); + raise_leader_floor( ctx, bound ); + ctx->failover_hist_adopted = 1; + ctx->adopted_tip = ULONG_MAX; + result.result = FD_VOTOR_ADOPT_SUCCESS; + result.root = ag_votor_metrics( ctx->votor ).highest_final_cert_slot; + result.vote_bound = ag_votor_get_vote_bound( ctx->votor ); + FD_LOG_NOTICE(( "failover: starting from an empty vote history on anchor %lu, no vote at or below slot %lu", result.root, result.vote_bound )); + return result; + } + + ulong tip = ag_hist_tip( hist ); + if( FD_UNLIKELY( tip==ULONG_MAX ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history on anchor %lu that has no votes in it", hist->anchor )); + return result; + } + + /* Reject a history older than the votes we sent as THIS identity. */ + result.result = FD_VOTOR_ADOPT_ERR_STALE; + if( FD_UNLIKELY( ctx->last_vote_slot!=ULONG_MAX && tiplast_vote_slot && fd_pubkey_eq( target, &ctx->last_vote_key ) ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history through slot %lu, this machine already voted as this identity through slot %lu", tip, ctx->last_vote_slot )); + return result; + } + + /* Reject a finality anchor past the blocks we replayed. */ + result.result = FD_VOTOR_ADOPT_ERR_UNREPLAYED_ROOT; + if( FD_UNLIKELY( ctx->highest_completed_slot && hist->anchor>ctx->highest_completed_slot ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history on anchor %lu, replay has only reached slot %lu, retry the promotion once replay passes the anchor", hist->anchor, ctx->highest_completed_slot )); + return result; + } + + ulong conflicts = ag_votor_hist_adopt( ctx->votor, hist ); + if( FD_UNLIKELY( conflicts==ULONG_MAX ) ) { + result.result = FD_VOTOR_ADOPT_ERR_CAPACITY; + FD_LOG_WARNING(( "failover: vote history exceeds the available slot state capacity, retry as consensus prunes old slots" )); + return result; + } + if( FD_UNLIKELY( conflicts ) ) FD_LOG_WARNING(( "adopted vote history disagrees with %lu of our own notar hashes, theirs were the ones sent", conflicts )); + ctx->failover_hist_adopted = 1; + ctx->adopted_tip = tip; + raise_leader_floor( ctx, hist->last_leader_slot ); + raise_leader_floor( ctx, ag_votor_get_vote_bound( ctx->votor ) ); + result.result = FD_VOTOR_ADOPT_SUCCESS; + result.root = ag_votor_metrics( ctx->votor ).highest_final_cert_slot; + result.vote_slot = tip; + result.vote_bound = ag_votor_get_vote_bound( ctx->votor ); + if( FD_LIKELY( result.vote_bound==ULONG_MAX ) ) FD_LOG_NOTICE(( "failover: adopted a vote history through slot %lu, the anchor is now %lu", tip, result.root )); + else FD_LOG_NOTICE(( "failover: adopted a vote history through slot %lu, the anchor is now %lu, no vote at or below slot %lu", tip, result.root, result.vote_bound )); + return result; +} + +/* Applies the failover voting rule at the end of an identity switch. */ +static void +failover_switched( fd_votor_tile_t * ctx ) { + /* Skip leader windows an adopted history says this identity may + already have led. Without one, set-identity keeps its plain rules. */ + ulong floor = leader_floor( ctx ); + if( FD_UNLIKELY( ctx->adopted_last_leader_slot!=ULONG_MAX ) ) { + ulong next_window = ag_first_slot_in_window( ctx->highest_parent_ready_slot )+AG_SLOTS_PER_WINDOW; + ulong floor_window = ag_first_slot_in_window( floor )+AG_SLOTS_PER_WINDOW; + if( FD_UNLIKELY( floor_window>next_window ) ) { + ctx->next_leader_slot = fd_multi_epoch_leaders_get_next_slot( ctx->mleaders, fd_ulong_max( fd_ulong_max( ctx->curr_epoch_slot, floor_window ), ctx->vote_history_slot ), &ctx->id_key ); + } + } + int operator = !FD_VOLATILE_CONST( ctx->id_keyswitch->failover ); + int junk = fd_pubkey_eq( &ctx->id_key, &ctx->junk_key ); + ctx->vote_authority = operator || ( !junk && ctx->failover_hist_adopted ); + ctx->installed_by_failover = !operator; + + /* A promotion votes past the adopted history's tip, its vote bound + and the leader floor, which are consensus state and lost on a + restart. The vote history file written at unhalt holds only the + watermark, so raise it to cover them and the file this identity + boots from after a crash waits past them too. The old identity's + file was written at the halt and is not affected. */ + + if( FD_UNLIKELY( ctx->vote_authority && !operator ) ) { + ulong slots[ 3 ] = { ctx->adopted_tip, ag_votor_get_vote_bound( ctx->votor ), floor }; + for( ulong i=0UL; i<3UL; i++ ) { + if( FD_LIKELY( slots[ i ]!=ULONG_MAX ) ) ctx->highest_vote_or_lead_slot = fd_ulong_max( ctx->highest_vote_or_lead_slot, slots[ i ] ); + } + } + if( FD_UNLIKELY( !operator ) ) { + FD_BASE58_ENCODE_32_BYTES( ctx->id_key.uc, id_key_b58 ); + FD_LOG_NOTICE(( "failover: switched to identity `%s`, this machine %s", id_key_b58, ctx->vote_authority ? "votes as it" : "does not vote as it" )); + if( FD_UNLIKELY( !junk && !ctx->failover_hist_adopted ) ) FD_LOG_WARNING(( "failover: the identity was installed without an adopted vote history, refusing to vote until an adoption precedes an identity switch" )); + } + ctx->failover_hist_adopted = 0; + ctx->adopted_tip = ULONG_MAX; + ctx->id_keyswitch->result = ctx->hist_seq; +} + +/* Records the slot replay completed and its root. */ +static void +failover_slot_completed( fd_votor_tile_t * ctx, + fd_replay_slot_completed_t const * slot_completed ) { + ctx->highest_completed_slot = fd_ulong_max( ctx->highest_completed_slot, slot_completed->slot ); + if( FD_LIKELY( slot_completed->root_slot!=ULONG_MAX ) ) ctx->root_slot = slot_completed->root_slot; +} + +/* Records a vote we built, sent or not, and publishes the history. A + vote that did not go out changes no consensus state here, its window + reaches the adopter in the exported vote bound. */ +static void +failover_after_vote( fd_votor_tile_t * ctx, + fd_stem_context_t * stem, + ulong vote_slot, + int sent ) { + if( FD_LIKELY( sent ) ) { + if( FD_LIKELY( ctx->last_vote_slot==ULONG_MAX || vote_slot>ctx->last_vote_slot ) ) { + ctx->last_vote_slot = vote_slot; + ctx->last_vote_key = ctx->id_key; + } + } else if( FD_UNLIKELY( ctx->highest_unsent_slot==ULONG_MAX || vote_slot>ctx->highest_unsent_slot ) ) { + ctx->highest_unsent_slot = vote_slot; + } + publish_hist( ctx, stem, sent ); +} + +/* Records our LEADER and publishes the history. */ +static void +failover_after_leader( fd_votor_tile_t * ctx, + fd_stem_context_t * stem ) { + ctx->last_leader_slot = ctx->next_leader_slot; + publish_hist( ctx, stem, 0 ); +} + +/* failover_init initializes failover in the votor tile, all of it off + unless the topology has the failover links. */ +static void +failover_init( fd_votor_tile_t * ctx, + fd_topo_t const * topo, + fd_topo_tile_t const * tile ) { + /* Without the failover links failover stays off. */ + ulong failov_in_idx = fd_topo_find_tile_in_link ( topo, tile, "failov_votor", tile->kind_id ); + ctx->hist_out_idx = fd_topo_find_tile_out_link( topo, tile, "votor_hist", tile->kind_id ); + ctx->failov_out_idx = fd_topo_find_tile_out_link( topo, tile, "votor_failov", tile->kind_id ); + ctx->failover_enabled = failov_in_idx!=ULONG_MAX; + if( FD_UNLIKELY( ( ctx->hist_out_idx!=ULONG_MAX )!=ctx->failover_enabled || ( ctx->failov_out_idx!=ULONG_MAX )!=ctx->failover_enabled ) ) { + FD_LOG_ERR(( "votor tile needs all of the failov_votor, votor_hist and votor_failov links or none of them" )); + } + if( FD_UNLIKELY( ctx->failover_enabled ) ) { + fd_topo_link_t const * hist_out = &topo->links[ tile->out_link_id[ ctx->hist_out_idx ] ]; + ctx->hist_out_mem = topo->workspaces[ topo->objs[ hist_out->dcache_obj_id ].wksp_id ].wksp; + ctx->hist_out_chunk0 = fd_dcache_compact_chunk0( ctx->hist_out_mem, hist_out->dcache ); + ctx->hist_out_wmark = fd_dcache_compact_wmark ( ctx->hist_out_mem, hist_out->dcache, hist_out->mtu ); + ctx->hist_out_chunk = ctx->hist_out_chunk0; + fd_topo_link_t const * failov_out = &topo->links[ tile->out_link_id[ ctx->failov_out_idx ] ]; + ctx->failov_out_mem = topo->workspaces[ topo->objs[ failov_out->dcache_obj_id ].wksp_id ].wksp; + ctx->failov_out_chunk0 = fd_dcache_compact_chunk0( ctx->failov_out_mem, failov_out->dcache ); + ctx->failov_out_wmark = fd_dcache_compact_wmark ( ctx->failov_out_mem, failov_out->dcache, failov_out->mtu ); + ctx->failov_out_chunk = ctx->failov_out_chunk0; + } + + /* Every validator votes as its boot identity with the usual voting + rules. Only an adopted history changes them, on the machine that + adopted it. */ + fd_memcpy( ctx->junk_key.uc, topo->failover_junk, 32UL ); + ctx->installed_by_failover = 0; + ctx->vote_authority = 1; + ctx->highest_unsent_slot = ULONG_MAX; + ctx->failover_hist_adopted = 0; + ctx->adopted_tip = ULONG_MAX; + ctx->hist_was_truncated = 0; + ctx->last_leader_slot = ULONG_MAX; + ctx->adopted_last_leader_slot = ULONG_MAX; + ctx->last_vote_slot = ULONG_MAX; + ctx->root_slot = ULONG_MAX; + ctx->adopt_req_sz = 0UL; + ctx->hist_seq = 0UL; + ctx->highest_completed_slot = 0UL; +} + +/* Copies an adoption request from the failover tile, after_frag responds + to it. */ +static void +failover_adopt_copy( fd_votor_tile_t * ctx, + ulong in_idx, + ulong chunk, + ulong sz ) { + ctx->adopt_req_sz = sz; + if( FD_LIKELY( sz<=sizeof(ctx->adopt_req) ) ) fd_memcpy( ctx->adopt_req, fd_chunk_to_laddr_const( ctx->in[ in_idx ].mem, chunk ), sz ); +} + +/* Responds to an adoption request, echoing its sequence number. */ +static void +failover_adopt_response( fd_votor_tile_t * ctx, + ulong sig, + fd_stem_context_t * stem ) { + fd_votor_adopt_result_t result; + if( FD_UNLIKELY( ctx->adopt_req_szadopt_req_sz>sizeof(ctx->adopt_req) ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history adoption of %lu bytes, it has no target identity", ctx->adopt_req_sz )); + result = (fd_votor_adopt_result_t){ .result=FD_VOTOR_ADOPT_ERR_DECODE, .root=ULONG_MAX, .vote_slot=ULONG_MAX, .vote_bound=ULONG_MAX }; + } else if( FD_UNLIKELY( ctx->vote_authority && ctx->installed_by_failover ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history adoption, this machine votes as an identity failover handed it" )); + result = (fd_votor_adopt_result_t){ .result=FD_VOTOR_ADOPT_ERR_INVALID, .root=ULONG_MAX, .vote_slot=ULONG_MAX, .vote_bound=ULONG_MAX }; + } else { + ulong boot_bound = ULONG_MAX; + ag_hist_t local; + ag_votor_hist_export( ctx->votor, ULONG_MAX, &local ); + ulong tip = ag_hist_tip( &local ); + if( tip!=ULONG_MAX ) boot_bound = ag_first_slot_in_window( tip )+AG_SLOTS_PER_WINDOW-1UL; + result = failover_adopt_hist( ctx, (fd_pubkey_t const *)ctx->adopt_req, ctx->adopt_req+FD_VOTOR_ADOPT_REQ_HDR_SZ, + ctx->adopt_req_sz-FD_VOTOR_ADOPT_REQ_HDR_SZ ); + if( FD_LIKELY( result.result==FD_VOTOR_ADOPT_SUCCESS ) ) { + if( boot_bound!=ULONG_MAX ) { + ag_votor_set_vote_bound( ctx->votor, boot_bound ); + raise_leader_floor( ctx, boot_bound ); + result.vote_bound = ag_votor_get_vote_bound( ctx->votor ); + } + ctx->vote_authority = 0; + update_own_rank( ctx ); + update_own_bls_key( ctx ); + } + } + fd_memcpy( fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk ), &result, sizeof(result) ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->failov_out_idx, sig, ctx->failov_out_chunk, sizeof(result), 0UL, tspub, tspub ); + ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(result), ctx->failov_out_chunk0, ctx->failov_out_wmark ); +} + static void handle_epoch( fd_votor_tile_t * ctx, fd_epoch_info_msg_t const * msg ) { @@ -1485,6 +1838,7 @@ handle_epoch( fd_votor_tile_t * ctx, ctx->ns_per_slot = (long)msg->ns_per_slot; ctx->slots_per_epoch = msg->epoch_schedule.slots_per_epoch; + if( FD_UNLIKELY( ctx->failover_enabled && !ctx->vote_authority ) ) own_rank = USHORT_MAX; ag_pool_advance_epoch ( ctx->pool, epoch_info, own_rank, msg->start_slot ); ag_votor_advance_epoch( ctx->votor, ctx->ns_per_slot, own_rank, msg->start_slot, NULL ); update_own_bls_key( ctx ); @@ -1566,6 +1920,7 @@ handle_replay( fd_votor_tile_t * ctx, fd_replay_slot_completed_t const * slot_completed = &replay->slot_completed; ag_block_id_t block_id = ag_block_id( slot_completed->slot, slot_completed->block_id.uc ); ag_block_id_t parent_block_id = ag_block_id( slot_completed->parent_slot, slot_completed->parent_block_id.uc ); + if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_slot_completed( ctx, slot_completed ); if( FD_UNLIKELY( ag_pool_finalized_slot( ctx->pool )==ULONG_MAX ) ) { ctx->boot_block_id = block_id; @@ -1844,6 +2199,7 @@ during_housekeeping( fd_votor_tile_t * ctx ) { ulong next_window = ag_first_slot_in_window( ctx->highest_parent_ready_slot )+AG_SLOTS_PER_WINDOW; ctx->next_leader_slot = fd_multi_epoch_leaders_get_next_slot( ctx->mleaders, fd_ulong_max( fd_ulong_max( ctx->curr_epoch_slot, next_window ), ctx->vote_history_slot ), &ctx->id_key ); + if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_switched( ctx ); fd_keyswitch_state( ctx->id_keyswitch, FD_KEYSWITCH_STATE_COMPLETED ); } } @@ -2076,7 +2432,8 @@ after_credit( fd_votor_tile_t * ctx, ctx->next_leader_slot = fd_multi_epoch_leaders_get_next_slot( ctx->mleaders, ag_first_slot_in_window( ctx->next_leader_slot )+AG_SLOTS_PER_WINDOW, &ctx->id_key ); } ag_block_id_t parent = { .slot = ULONG_MAX }; /* never will be leader, schedule exhausted, or the pool has not granted parent ready yet */ - if( FD_LIKELY( ctx->next_leader_slot!=ULONG_MAX ) ) parent = ag_pool_wait_for_parent_ready( ctx->pool, ctx->next_leader_slot ); + if( FD_LIKELY( ctx->next_leader_slot!=ULONG_MAX && ( !ctx->failover_enabled || ctx->vote_authority ) ) ) + parent = ag_pool_wait_for_parent_ready( ctx->pool, ctx->next_leader_slot ); if( FD_UNLIKELY( parent.slot!=ULONG_MAX && !ctx->halt_signing ) ) { ctx->highest_vote_or_lead_slot = fd_ulong_max( ctx->highest_vote_or_lead_slot, ag_first_slot_in_window( ctx->next_leader_slot )+AG_SLOTS_PER_WINDOW-1UL ); if( FD_LIKELY( ctx->vote_history_fd_dir!=-1 ) ) write_vote_history( ctx ); /* a restart does not lead the window again */ @@ -2088,6 +2445,7 @@ after_credit( fd_votor_tile_t * ctx, ulong reward_slot = fd_ulong_sat_sub( ctx->next_leader_slot, FD_NUM_SLOTS_FOR_REWARD ); for( ulong i=0UL; ifailover_enabled ) ) failover_after_leader( ctx, stem ); ctx->next_leader_slot = fd_multi_epoch_leaders_get_next_slot( ctx->mleaders, ag_first_slot_in_window( ctx->next_leader_slot )+AG_SLOTS_PER_WINDOW, &ctx->id_key ); *charge_busy = 1; } @@ -2265,7 +2623,8 @@ after_credit( fd_votor_tile_t * ctx, ag_epoch_info_t const * epoch_info = fd_ptr_if( vote_slot>=ctx->next_epoch_slot, ctx->next_epoch_info, fd_ptr_if( vote_slot>=ctx->curr_epoch_slot, ctx->curr_epoch_info, ctx->prev_epoch_info ) ); ulong rank = ag_vote_rank( &ctx->scratch.vote ); ctx->highest_vote_or_lead_slot = fd_ulong_max( ctx->highest_vote_or_lead_slot, vote_slot ); - if( FD_LIKELY( epoch_info && rankvalidator_cnt && vote_slot>=ctx->vote_history_slot && vote_slot>=ctx->catchup_vote_slot && !blst_p2_affine_is_inf( ag_vote_sig( &ctx->scratch.vote ) ) ) ) { + int sent = epoch_info && rankvalidator_cnt && vote_slot>=ctx->vote_history_slot && vote_slot>=ctx->catchup_vote_slot && !blst_p2_affine_is_inf( ag_vote_sig( &ctx->scratch.vote ) ); + if( FD_LIKELY( sent ) ) { if( FD_LIKELY( ctx->vote_history_fd_dir!=-1 ) ) write_vote_history( ctx ); ulong ser_sz = ag_vote_ser( &ctx->scratch.vote, ctx->scratch.ser ); long broadcast_start_time = fd_clock_tile_now( ctx->clock ); @@ -2319,6 +2678,7 @@ after_credit( fd_votor_tile_t * ctx, report_alpenglow_vote( ctx, NULL, &ctx->scratch.vote, ctx->scratch.ser[ 1 ], result, quorum_reached, FD_EVENT_ALPENGLOW_VOTE_REASON_BLOCK_REPLAYED+reason, aggregation_start_time, broadcast_start_time, reward_leader ); if( FD_UNLIKELY( reward_leader && ( !reward_leader->tx_conn || reward_leader->tx_conn->state!=FD_QUIC_CONN_STATE_ACTIVE ) ) ) report_alpenglow_vote( ctx, NULL, &ctx->scratch.vote, ctx->scratch.ser[ 1 ], result, 0, FD_EVENT_ALPENGLOW_VOTE_REASON_REWARD_NO_CONNECTION, 0L, 0L, reward_leader ); } + if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_after_vote( ctx, stem, vote_slot, sent ); *opt_poll_in = 0; *charge_busy = 1; } @@ -2383,6 +2743,11 @@ before_frag( fd_votor_tile_t * ctx, if( FD_UNLIKELY( !ctx->curr_epoch_info ) ) return -1; ctx->replay_in_seq = seq+1UL; return 0; + case IN_KIND_FAILOV: + /* Drain votes and pool events produced under the current identity + before changing the history or removing its rank. */ + if( FD_UNLIKELY( ctx->init && ( ag_votor_metrics( ctx->votor ).vote_events_cnt || ag_pool_metrics( ctx->pool ).pool_events_cnt ) ) ) return -1; + return 0; default: FD_LOG_ERR(( "unexpected in_kind %d", ctx->in_kind[ in_idx ] )); } @@ -2414,6 +2779,9 @@ during_frag( fd_votor_tile_t * ctx, case IN_KIND_REPLAY: handle_replay( ctx, sig, fd_chunk_to_laddr_const( ctx->in[ in_idx ].mem, chunk ) ); break; + case IN_KIND_FAILOV: + failover_adopt_copy( ctx, in_idx, chunk, sz ); + break; default: FD_LOG_ERR(( "unexpected in_kind %d", ctx->in_kind[ in_idx ] )); } @@ -2466,6 +2834,10 @@ after_frag( fd_votor_tile_t * ctx, } case IN_KIND_REPLAY: /* reliable link, handled in during_frag */ + if( FD_UNLIKELY( ctx->failover_enabled && sig==REPLAY_SIG_SLOT_COMPLETED ) ) publish_hist( ctx, stem, 0 ); /* keeps the failover deadlines ticking */ + break; + case IN_KIND_FAILOV: + failover_adopt_response( ctx, sig, stem ); break; default: FD_LOG_ERR(( "unexpected in_kind %d", ctx->in_kind[ in_idx ] )); @@ -2513,18 +2885,19 @@ privileged_init( fd_topo_t const * topo, another name, like the .bak a set-identity away from it saves, or the file an earlier version renamed to the new identity. */ + int should_scan = strcmp( tile->votor.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ); if( FD_UNLIKELY( -1==faccessat( ctx->vote_history_fd_dir, bin, F_OK, 0 ) ) ) { if( FD_UNLIKELY( errno!=ENOENT ) ) FD_LOG_ERR(( "faccessat(`%s/%s`) failed (%i-%s)", tile->votor.vote_history_path, bin, errno, fd_io_strerror( errno ) )); - DIR * dir = opendir( tile->votor.vote_history_path ); - if( FD_UNLIKELY( !dir ) ) FD_LOG_ERR(( "opendir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); + DIR * dir = should_scan ? opendir( tile->votor.vote_history_path ) : NULL; + if( FD_UNLIKELY( should_scan && !dir ) ) FD_LOG_ERR(( "opendir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); errno = 0; - for( struct dirent * ent; (ent = readdir( dir )); ) { + for( struct dirent * ent; dir && (ent = readdir( dir )); ) { if( FD_LIKELY( strncmp( ent->d_name, "vote_history-", sizeof("vote_history-")-1UL ) ) ) continue; FD_LOG_ERR(( "Booting as %s, but detected %s in %s, which is missing %s. Please either copy %s into %s or remove the vote_history-* files there if you wish to boot without a vote history file.", id_key_b58, ent->d_name, tile->votor.vote_history_path, bin, bin, tile->votor.vote_history_path )); } if( FD_UNLIKELY( errno ) ) FD_LOG_ERR(( "readdir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); - if( FD_UNLIKELY( closedir( dir ) ) ) FD_LOG_ERR(( "closedir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( dir && closedir( dir ) ) ) FD_LOG_ERR(( "closedir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); FD_LOG_WARNING(( "vote history file `%s/%s` does not exist, creating it. This identity is assumed to have never voted.", tile->votor.vote_history_path, bin )); } ctx->vote_history_slot = read_vote_history( ctx, tile->votor.vote_history_path, bin ); @@ -2676,6 +3049,7 @@ unprivileged_init( fd_topo_t const * topo, } else if( FD_LIKELY( !strcmp( link->name, "replay_slot" ) ) ) ctx->in_kind[ i ] = IN_KIND_REPLAY; else if( FD_LIKELY( !strcmp( link->name, "sign_votor" ) ) ) ctx->in_kind[ i ] = IN_KIND_SIGN; + else if( FD_LIKELY( !strcmp( link->name, "failov_votor" ) ) ) ctx->in_kind[ i ] = IN_KIND_FAILOV; else FD_LOG_ERR(( "votor tile has unexpected input link %lu %s", i, link->name )); if( FD_LIKELY( link->mtu ) ) ctx->in[ i ].mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; @@ -2697,6 +3071,8 @@ unprivileged_init( fd_topo_t const * topo, ctx->net_out_wmark = fd_dcache_compact_wmark ( ctx->net_out_mem, net_out->dcache, net_out->mtu ); ctx->net_out_chunk = ctx->net_out_chunk0; + failover_init( ctx, topo, tile ); + ulong sign_in_idx = fd_topo_find_tile_in_link ( topo, tile, "sign_votor", tile->kind_id ); ulong sign_out_idx = fd_topo_find_tile_out_link( topo, tile, "votor_sign", tile->kind_id ); FD_TEST( sign_in_idx !=ULONG_MAX ); diff --git a/src/discof/votor/fd_votor_tile.h b/src/discof/votor/fd_votor_tile.h index 9eac4d00f65..be29d907d3f 100644 --- a/src/discof/votor/fd_votor_tile.h +++ b/src/discof/votor/fd_votor_tile.h @@ -2,6 +2,7 @@ #define HEADER_fd_src_discof_votor_fd_votor_tile_h #include "../../ballet/bls/fd_bls.h" +#include "../../choreo/votor/ag_hist.h" #include "../../disco/topo/fd_topo.h" #include "../../waltz/quic/tls/fd_quic_tls.h" @@ -126,6 +127,48 @@ union fd_votor_msg { }; typedef union fd_votor_msg fd_votor_msg_t; +/* fd_votor_hist_msg is one frame on the votor_hist link to the failover + tile, with our replay slots and our own vote history. */ + +#define FD_VOTOR_HIST_SIG (0UL) + +struct fd_votor_hist_msg { + fd_pubkey_t identity; /* identity installed when this history was published */ + ulong replay_slot; /* highest slot replay completed, ULONG_MAX before any */ + ulong root_slot; /* replay's root at the last completed slot, ULONG_MAX before any */ + ulong vote_slot; /* the history's tip, ULONG_MAX while it is empty */ + int has_vote; /* we sent a vote just before this frame, so vote_slot is a vote we signed */ + int truncated; /* the export dropped old windows and lifted the anchor */ + ag_hist_t hist; +}; +typedef struct fd_votor_hist_msg fd_votor_hist_msg_t; + +/* Adoption of a peer's vote history. */ + +/* Frame sizes */ + +#define FD_VOTOR_ADOPT_REQ_HDR_SZ (32UL) /* the identity the adoption is for, before the payload */ +#define FD_VOTOR_ADOPT_EMPTY_SZ (8UL) /* an empty payload, a slot as a ulong */ + +/* Result codes */ + +#define FD_VOTOR_ADOPT_SUCCESS (0UL) +#define FD_VOTOR_ADOPT_ERR_DECODE (1UL) +#define FD_VOTOR_ADOPT_ERR_INVALID (2UL) +#define FD_VOTOR_ADOPT_ERR_UNREPLAYED_ROOT (3UL) /* the history's finality anchor is past our replayed slots */ +#define FD_VOTOR_ADOPT_ERR_UNREPLAYED (4UL) /* consensus is not up yet, ask again as replay moves */ +#define FD_VOTOR_ADOPT_ERR_STALE (5UL) /* older than the votes this identity sent from here */ +#define FD_VOTOR_ADOPT_ERR_CAPACITY (6UL) /* retained local and imported slots exceed capacity */ +#define FD_VOTOR_ADOPT_RESULT_CNT (7UL) + +struct fd_votor_adopt_result { + ulong result; + ulong root; /* our finality anchor after the adoption */ + ulong vote_slot; /* the adopted history's tip, ULONG_MAX for an empty one */ + ulong vote_bound; /* we cast no vote at or below this slot, ULONG_MAX when there is none */ +}; +typedef struct fd_votor_adopt_result fd_votor_adopt_result_t; + extern fd_topo_run_tile_t fd_tile_votor; #endif /* HEADER_fd_src_discof_votor_fd_votor_tile_h */ diff --git a/src/discof/votor/test_votor_tile.c b/src/discof/votor/test_votor_tile.c index 49ec2122015..50ade04c4a3 100644 --- a/src/discof/votor/test_votor_tile.c +++ b/src/discof/votor/test_votor_tile.c @@ -1742,6 +1742,44 @@ test_gossip_before_stake( void ) { test_ctx_delete( &ctx ); } +/* A tile ranked 0 in a three-voter epoch with consensus up at slot 0, + failover on, booted under its own key, for the history tests. */ + +static ag_epoch_info_t * +hist_setup( fd_votor_tile_t * ctx ) { + memset( ctx, 0, sizeof(fd_votor_tile_t) ); + ctx->vote_history_fd_dir = -1; + fd_vote_stake_weight_t stakes[ TEST_VOTER_MAX ]; + build_stakes( stakes, 3UL, 10UL ); + ag_epoch_info_t * epoch_info = rank_voters( &epoch_info_mem, stakes, 3UL, NULL, NULL ); + memcpy( ctx->id_key.uc, epoch_info->validators[0].id_key, sizeof(fd_pubkey_t) ); + ctx->junk_key = ctx->id_key; + ctx->curr_epoch_info = epoch_info; + ctx->failover_enabled = 1; + ctx->hist_out_idx = ULONG_MAX; + ctx->failov_out_idx = ULONG_MAX; + ctx->last_leader_slot = ULONG_MAX; + ctx->adopted_last_leader_slot = ULONG_MAX; + ctx->last_vote_slot = ULONG_MAX; + ctx->highest_unsent_slot = ULONG_MAX; + ctx->adopted_tip = ULONG_MAX; + ctx->root_slot = ULONG_MAX; + + ctx->pool = test_pool( epoch_info, 0UL ); + ctx->votor = ag_votor_join( ag_votor_new( votor_scratch, 64UL, 42UL ) ); + FD_TEST( ctx->votor ); + ag_votor_init ( ctx->votor, &(ag_block_id_t){ .slot = 0UL }, 0L, 400000000L, (ushort)1, capture_sign_bls, NULL ); + ag_votor_advance_epoch( ctx->votor, 400000000L, 0UL, 0UL, epoch_info->validators[0].bls_key ); + ctx->init = 1; + return epoch_info; +} + +static void +hist_teardown( fd_votor_tile_t * ctx ) { + ag_votor_delete( ag_votor_leave( ctx->votor ) ); + ag_pool_delete( ag_pool_leave( ctx->pool ) ); +} + /* after_credit connects queued peers once due and leaves the rest, requeues one whose backoff grew, and drops entries for peers that can no longer be connected. */ @@ -2619,6 +2657,380 @@ test_reward_certs( void ) { FD_LOG_NOTICE(( "pass: test_reward_certs" )); } +/* Test that an adopted history raises the leader floor, and a lower one + leaves it alone. */ + +static void +test_history_raises_the_leader_floor( void ) { + static fd_votor_tile_t ctx; + hist_setup( &ctx ); + + ag_hist_t hist = { .anchor = 0UL, .last_leader_slot = 12UL, .vote_bound = 17UL, .rec_cnt = 1UL }; + hist.rec[ 0 ].slot = 1UL; + hist.rec[ 0 ].flags = AG_HIST_FLAG_VOTED; + uchar req[ AG_HIST_SER_MAX ]; + ulong req_sz; + FD_TEST( !ag_hist_ser( &hist, req, sizeof(req), &req_sz ) ); + fd_votor_adopt_result_t result = failover_adopt_hist( &ctx, &ctx.id_key, req, req_sz ); + FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS && result.vote_slot==1UL && result.vote_bound==17UL ); + FD_TEST( ag_votor_get_vote_bound( ctx.votor )==17UL ); + FD_TEST( ctx.adopted_last_leader_slot==ag_first_slot_in_window( 17UL ) ); + FD_TEST( leader_floor( &ctx )==ag_first_slot_in_window( 17UL ) ); + + hist.last_leader_slot = 4UL; + hist.vote_bound = 6UL; + FD_TEST( !ag_hist_ser( &hist, req, sizeof(req), &req_sz ) ); + result = failover_adopt_hist( &ctx, &ctx.id_key, req, req_sz ); + FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS && result.vote_bound==17UL ); + FD_TEST( ctx.adopted_last_leader_slot==ag_first_slot_in_window( 17UL ) ); + + uchar empty[ FD_VOTOR_ADOPT_EMPTY_SZ ]; + FD_STORE( ulong, empty, 2UL ); + result = failover_adopt_hist( &ctx, &ctx.id_key, empty, sizeof(empty) ); + FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS && result.vote_bound==17UL ); + FD_TEST( ctx.adopted_last_leader_slot==ag_first_slot_in_window( 17UL ) ); + + /* A history older than the votes this machine sent as that identity + is rejected, one for another identity is not. */ + ctx.last_vote_slot = 3UL; + ctx.last_vote_key = ctx.id_key; + result = failover_adopt_hist( &ctx, &ctx.id_key, req, req_sz ); + FD_TEST( result.result==FD_VOTOR_ADOPT_ERR_STALE ); + fd_pubkey_t other; fd_memset( other.uc, 0x77, 32UL ); + result = failover_adopt_hist( &ctx, &other, req, req_sz ); + FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS ); + + hist_teardown( &ctx ); +} + +/* Test that the votor behaves as usual when the failover links are + absent. */ + +static void +test_failover_off( void ) { + static fd_votor_tile_t ctx; + static fd_topo_t topo[1]; + static fd_topo_tile_t tile[1]; + static auth_vtr_t auth_vtr_mem[ 1UL<validators[0].id_key, sizeof(fd_pubkey_t) ); + failover_init( &ctx, topo, tile ); + FD_TEST( !ctx.failover_enabled && ctx.vote_authority ); + FD_TEST( ctx.hist_out_idx==ULONG_MAX && ctx.failov_out_idx==ULONG_MAX ); + + ag_bls_key_t bls_key; memcpy( bls_key, epoch_info->validators[0].bls_key, sizeof(ag_bls_key_t) ); + init_keys( &ctx, auth_vtr_mem, &bls_key, 1UL ); + ctx.curr_epoch_info = epoch_info; + ctx.next_epoch_slot = ULONG_MAX; + ctx.next_leader_slot = ULONG_MAX; + ctx.pool = test_pool( epoch_info, 0UL ); + ctx.votor = ag_votor_join( ag_votor_new( votor_scratch, 64UL, 42UL ) ); + FD_TEST( ctx.votor ); + ag_votor_init ( ctx.votor, &(ag_block_id_t){ .slot = 0UL }, 0L, 400000000L, (ushort)1, capture_sign_bls, NULL ); + ag_votor_advance_epoch( ctx.votor, 400000000L, 0UL, 0UL, NULL ); /* no key until update_own_bls_key picks ours */ + update_own_rank( &ctx ); + update_own_bls_key( &ctx ); + ctx.init = 1; + + /* We lead every slot of epoch 0. */ + + ctx.mleaders = fd_multi_epoch_leaders_join( fd_multi_epoch_leaders_new( mleaders_mem ) ); + ctx.peers = peers_join( peers_new( peers_mem, PEERS_LG_SLOT_CNT, 42UL ) ); + ctx.reconn_prq = reconn_prq_join( reconn_prq_new( reconn_prq_mem, RECONN_MAX ) ); + FD_TEST( ctx.mleaders && ctx.peers && ctx.reconn_prq ); + static uchar stake_msg_mem[ FD_STAKE_CI_STAKE_MSG_HEADER_SZ+FD_STAKE_CI_STAKE_MSG_RECORD_SZ ] __attribute__((aligned(8))); + fd_stake_weight_msg_t * stake_msg = fd_type_pun( stake_msg_mem ); + *stake_msg = (fd_stake_weight_msg_t){ .epoch = 0UL, .staked_vote_cnt = 1UL, .start_slot = 0UL, .slot_cnt = 64UL }; + *(fd_vote_stake_weight_t *)fd_type_pun( stake_msg+1 ) = (fd_vote_stake_weight_t){ .vote_key = ctx.id_key, .id_key = ctx.id_key, .stake = 10UL }; + fd_multi_epoch_leaders_stake_msg_init( ctx.mleaders, stake_msg ); + fd_multi_epoch_leaders_stake_msg_fini( ctx.mleaders ); + fd_aio_t * aio = fd_aio_join( fd_aio_new( aio_mem, NULL, drop_aio_send ) ); + ctx.quic_client = test_quic( quic_client_scratch, sizeof(quic_client_scratch), FD_QUIC_ROLE_CLIENT, &ctx, aio ); + ctx.quic_server = test_quic( quic_server_scratch, sizeof(quic_server_scratch), FD_QUIC_ROLE_SERVER, &ctx, aio ); + ctx.auth_vtr_keyswitch = fd_keyswitch_join( fd_keyswitch_new( av_keyswitch_mem, FD_KEYSWITCH_STATE_UNLOCKED ) ); + ctx.id_keyswitch = fd_keyswitch_join( fd_keyswitch_new( id_keyswitch_mem, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.auth_vtr_keyswitch && ctx.id_keyswitch ); + fd_clock_tile_init( ctx.clock ); + ctx.vote_history_fd_dir = -1; + for( ulong i=0UL; ibytes, ctx.id_key.uc, sizeof(fd_pubkey_t) ); + ctx.id_keyswitch->result = 77UL; + fd_keyswitch_state( ctx.id_keyswitch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( ctx.id_keyswitch->state==FD_KEYSWITCH_STATE_COMPLETED ); + FD_TEST( ctx.next_leader_slot==8UL && ctx.vote_authority ); + FD_TEST( ctx.id_keyswitch->result==77UL ); + + fd_quic_delete( fd_quic_leave( fd_quic_fini( ctx.quic_client ) ) ); + fd_quic_delete( fd_quic_leave( fd_quic_fini( ctx.quic_server ) ) ); + reconn_prq_delete( reconn_prq_leave( ctx.reconn_prq ) ); + hist_teardown( &ctx ); +} + +/* Test that with failover a failover identity switch starts voting only + after adoption, and that set-identity votes as on main, even to the junk + key. */ + +static void +operator_switch( int operator, + ulong new_rank, + int expect_votes ) { + static fd_votor_tile_t ctx; + static auth_vtr_t auth_vtr_mem[ 1UL<validators[ new_rank ].bls_key, sizeof(ag_bls_key_t) ); + init_keys( &ctx, auth_vtr_mem, &bls_key, 1UL ); /* the new identity's key, as the resume loads it */ + ctx.id_keyswitch = fd_keyswitch_join( fd_keyswitch_new( id_keyswitch_mem, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); + FD_TEST( ctx.id_keyswitch ); + ctx.id_keyswitch->failover = (ulong)!operator; + ctx.hist_seq = 77UL; + + memcpy( ctx.id_key.uc, epoch_info->validators[ new_rank ].id_key, sizeof(fd_pubkey_t) ); + failover_switched( &ctx ); + FD_TEST( ctx.vote_authority==expect_votes && !ctx.failover_hist_adopted ); + FD_TEST( ctx.id_keyswitch->result==77UL ); + update_own_rank( &ctx ); + update_own_bls_key( &ctx ); + + ag_block_info_t block = {0}; + memset( block.hash, 1, sizeof(ag_block_hash_t) ); + ag_votor_process_replay( ctx.votor, 1UL, &block ); + ag_vote_t vote; + uchar reason; + FD_TEST( ag_votor_poll_vote( ctx.votor, &vote, &reason )==expect_votes ); + if( expect_votes ) FD_TEST( ag_vote_rank( &vote )==new_rank ); + + hist_teardown( &ctx ); +} + +/* With failover on, the end of the switch applies the failover rule and + leaves the history watermark for the admin tile. */ + +static void +test_switch_runs_the_failover_rule( void ) { + static fd_votor_tile_t ctx; + static auth_vtr_t auth_vtr_mem[ 1UL<validators[0].bls_key, sizeof(ag_bls_key_t) ); + init_keys( &ctx, auth_vtr_mem, &bls_key, 1UL ); + ctx.vote_authority = 1; + ctx.hist_seq = 77UL; + ctx.mleaders = fd_multi_epoch_leaders_join( fd_multi_epoch_leaders_new( mleaders_mem ) ); + ctx.peers = peers_join( peers_new( peers_mem, PEERS_LG_SLOT_CNT, 42UL ) ); + FD_TEST( ctx.mleaders && ctx.peers ); + fd_aio_t * aio = fd_aio_join( fd_aio_new( aio_mem, NULL, drop_aio_send ) ); + ctx.quic_client = test_quic( quic_client_scratch, sizeof(quic_client_scratch), FD_QUIC_ROLE_CLIENT, &ctx, aio ); + ctx.quic_server = test_quic( quic_server_scratch, sizeof(quic_server_scratch), FD_QUIC_ROLE_SERVER, &ctx, aio ); + ctx.auth_vtr_keyswitch = fd_keyswitch_join( fd_keyswitch_new( av_keyswitch_mem, FD_KEYSWITCH_STATE_UNLOCKED ) ); + ctx.id_keyswitch = fd_keyswitch_join( fd_keyswitch_new( id_keyswitch_mem, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.auth_vtr_keyswitch && ctx.id_keyswitch ); + fd_clock_tile_init( ctx.clock ); + + /* A failover switch to the junk key we booted under, no adoption. */ + memcpy( ctx.id_keyswitch->bytes, ctx.junk_key.uc, sizeof(fd_pubkey_t) ); + ctx.id_keyswitch->failover = 1UL; + fd_keyswitch_state( ctx.id_keyswitch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( ctx.id_keyswitch->state==FD_KEYSWITCH_STATE_COMPLETED ); + FD_TEST( !ctx.vote_authority && ctx.id_keyswitch->result==77UL ); + + hist_teardown( &ctx ); +} + +static void +test_operator_switch_votes( void ) { + operator_switch( 0, 1UL, 0 ); /* staked, no adoption */ + operator_switch( 1, 1UL, 1 ); /* staked, by set-identity */ + operator_switch( 1, 0UL, 1 ); /* to the junk key, by set-identity */ +} + +/* Test that an adoption is rejected while this machine votes as an + identity failover handed it and taken once it stands by again. */ + +static void +test_adopt_refused_while_voting( void ) { + static fd_votor_tile_t ctx; + hist_setup( &ctx ); + ctx.vote_authority = 1; + ctx.installed_by_failover = 1; + + static uchar reply_mem[ 512 ] __attribute__((aligned(FD_CHUNK_SZ))); + static fd_frag_meta_t mcache[ 8 ]; + fd_frag_meta_t * mcaches[] = { mcache }; + ulong seqs[] = { 0UL }, depths[] = { 8UL }; + ulong cr_avail = 64UL, min_cr_avail = 64UL; + int reliable = 0; + fd_stem_context_t stem[ 1 ] = {{ + .mcaches=mcaches, .seqs=seqs, .depths=depths, + .cr_avail=&cr_avail, .min_cr_avail=&min_cr_avail, + .cr_decrement_amount=1UL, .out_reliable=&reliable + }}; + ctx.failov_out_idx = 0UL; + ctx.failov_out_mem = reply_mem; + ctx.failov_out_chunk0 = 0UL; + ctx.failov_out_wmark = 4UL; + ctx.failov_out_chunk = 0UL; + ctx.in_kind[ 0 ] = IN_KIND_FAILOV; + fd_memcpy( ctx.adopt_req, ctx.id_key.uc, FD_VOTOR_ADOPT_REQ_HDR_SZ ); + FD_STORE( ulong, ctx.adopt_req+FD_VOTOR_ADOPT_REQ_HDR_SZ, 9UL ); /* empty history, no vote at or below slot 9 */ + ctx.adopt_req_sz = FD_VOTOR_ADOPT_REQ_HDR_SZ+FD_VOTOR_ADOPT_EMPTY_SZ; + + after_frag( &ctx, 0UL, 0UL, 901UL, ctx.adopt_req_sz, 0UL, 0UL, stem ); + fd_votor_adopt_result_t const * reply = fd_chunk_to_laddr_const( reply_mem, mcache[0].chunk ); + FD_TEST( seqs[0]==1UL && mcache[0].sig==901UL ); + FD_TEST( reply->result==FD_VOTOR_ADOPT_ERR_INVALID && !ctx.failover_hist_adopted ); + FD_TEST( ag_votor_get_vote_bound( ctx.votor )==ULONG_MAX ); + + /* A client retains its old authority if the imported slots cannot fit. */ + ctx.installed_by_failover = 0; + ag_hist_t hist = { .anchor = 12UL, .last_leader_slot = 80UL, .vote_bound = 83UL, .rec_cnt = AG_HIST_MAX }; + for( ulong i=0UL; iresult==FD_VOTOR_ADOPT_ERR_CAPACITY && !ctx.failover_hist_adopted ); + FD_TEST( ctx.vote_authority ); + FD_TEST( ctx.adopted_last_leader_slot==ULONG_MAX ); + FD_TEST( ag_votor_get_vote_bound( ctx.votor )==ULONG_MAX ); + FD_TEST( ag_votor_metrics( ctx.votor ).highest_final_cert_slot==0UL ); + FD_TEST( ag_votor_metrics( ctx.votor ).slot_state_pool_free==free ); + + ctx.vote_authority = 0; + FD_STORE( ulong, ctx.adopt_req+FD_VOTOR_ADOPT_REQ_HDR_SZ, 9UL ); + ctx.adopt_req_sz = FD_VOTOR_ADOPT_REQ_HDR_SZ+FD_VOTOR_ADOPT_EMPTY_SZ; + after_frag( &ctx, 0UL, 2UL, 903UL, ctx.adopt_req_sz, 0UL, 0UL, stem ); + reply = fd_chunk_to_laddr_const( reply_mem, mcache[2].chunk ); + FD_TEST( seqs[0]==3UL && mcache[2].sig==903UL ); + FD_TEST( reply->result==FD_VOTOR_ADOPT_SUCCESS && ctx.failover_hist_adopted ); + FD_TEST( ag_votor_get_vote_bound( ctx.votor )==9UL ); + + hist_teardown( &ctx ); +} + +/* Test that the exporter's floors and dropped votes reach the adopter + as the history's vote bound without changing the exporter's own + consensus state, so a validator that never adopts votes as one + without failover. */ + +static void +test_export_leaves_consensus_alone( void ) { + static fd_votor_tile_t ctx; + hist_setup( &ctx ); + + static uchar hist_mem[ 5UL*sizeof(fd_votor_hist_msg_t) ] __attribute__((aligned(FD_CHUNK_SZ))); + ulong hist_chunks = ( sizeof(fd_votor_hist_msg_t)+FD_CHUNK_SZ-1UL )/FD_CHUNK_SZ; + static fd_frag_meta_t mcache[ 8 ]; + fd_frag_meta_t * mcaches[] = { mcache }; + ulong seqs[] = { 0UL }, depths[] = { 8UL }; + ulong cr_avail = 64UL, min_cr_avail = 64UL; + int reliable = 0; + fd_stem_context_t stem[ 1 ] = {{ + .mcaches=mcaches, .seqs=seqs, .depths=depths, + .cr_avail=&cr_avail, .min_cr_avail=&min_cr_avail, + .cr_decrement_amount=1UL, .out_reliable=&reliable + }}; + ctx.hist_out_idx = 0UL; + ctx.hist_out_mem = hist_mem; + ctx.hist_out_chunk0 = 0UL; + ctx.hist_out_wmark = 3UL*hist_chunks; /* four frames, no wrap */ + ctx.hist_out_chunk = 0UL; + + /* A notar vote the tile built and sent. The root counts as voted, so + the history holds it and slot 1. */ + ag_block_info_t block = {0}; + memset( block.hash, 1, sizeof(ag_block_hash_t) ); + ag_votor_process_replay( ctx.votor, 1UL, &block ); + uchar reason; + FD_TEST( ag_votor_poll_vote( ctx.votor, &ctx.scratch.vote, &reason ) && ag_vote_slot( &ctx.scratch.vote )==1UL ); + failover_after_vote( &ctx, stem, 1UL, 1 ); + fd_votor_hist_msg_t const * msg = fd_chunk_to_laddr_const( hist_mem, mcache[ 0 ].chunk ); + FD_TEST( seqs[0]==1UL && msg->has_vote && msg->vote_slot==1UL && msg->hist.rec_cnt==2UL ); + FD_TEST( msg->hist.vote_bound==ULONG_MAX && ag_votor_get_vote_bound( ctx.votor )==ULONG_MAX ); + FD_TEST( ctx.last_vote_slot==1UL && ctx.highest_unsent_slot==ULONG_MAX ); + + /* A vote in slot 21 the tile did not send: the exported bound covers + its window, the library keeps no bound and no unsent mark. */ + failover_after_vote( &ctx, stem, 21UL, 0 ); + msg = fd_chunk_to_laddr_const( hist_mem, mcache[ 1 ].chunk ); + FD_TEST( seqs[0]==2UL && !msg->has_vote && msg->hist.vote_bound==23UL ); + FD_TEST( ctx.last_vote_slot==1UL && ctx.highest_unsent_slot==21UL ); + FD_TEST( ag_votor_get_vote_bound( ctx.votor )==ULONG_MAX ); + ag_hist_t local; + FD_TEST( !ag_votor_hist_export( ctx.votor, ULONG_MAX, &local ) ); + FD_TEST( local.rec_cnt==2UL && local.rec[ 1 ].slot==1UL && ( local.rec[ 1 ].flags & AG_HIST_FLAG_VOTED_NOTAR ) && !( local.rec[ 1 ].flags & AG_HIST_FLAG_BAD_WINDOW ) ); + + /* The restart and catch-up floors raise the exported bound the same + way, and only the highest window counts. */ + ctx.vote_history_slot = 30UL; /* its window is 28..31 */ + failover_after_leader( &ctx, stem ); + msg = fd_chunk_to_laddr_const( hist_mem, mcache[ 2 ].chunk ); + FD_TEST( seqs[0]==3UL && msg->hist.vote_bound==31UL ); + ctx.catchup_vote_slot = 9UL; + failover_after_leader( &ctx, stem ); + msg = fd_chunk_to_laddr_const( hist_mem, mcache[ 3 ].chunk ); + FD_TEST( seqs[0]==4UL && msg->hist.vote_bound==31UL ); + FD_TEST( ag_votor_get_vote_bound( ctx.votor )==ULONG_MAX ); + + hist_teardown( &ctx ); + FD_LOG_NOTICE(( "pass: test_export_leaves_consensus_alone" )); +} + int main( int argc, char ** argv ) { @@ -2663,6 +3075,12 @@ main( int argc, test_write_vote_history(); test_boot_vote_history_floor(); test_reward_certs(); + test_history_raises_the_leader_floor(); + test_failover_off(); + test_switch_runs_the_failover_rule(); + test_operator_switch_votes(); + test_adopt_refused_while_voting(); + test_export_leaves_consensus_alone(); FD_LOG_NOTICE(( "pass" )); fd_halt(); diff --git a/src/waltz/fd_token_bucket.h b/src/waltz/fd_token_bucket.h index fefa4b9fdf6..148946325f1 100644 --- a/src/waltz/fd_token_bucket.h +++ b/src/waltz/fd_token_bucket.h @@ -34,6 +34,19 @@ fd_token_bucket_consume( fd_token_bucket_t * bucket, return ok; } +/* fd_token_bucket_peek returns 1 if consuming delta at ts would + succeed without changing the bucket. */ + +FD_FN_PURE static inline int +fd_token_bucket_peek( fd_token_bucket_t const * bucket, + float delta, + long ts ) { + long elapsed = ts - bucket->ts; + float balance = bucket->balance + ((float)elapsed * bucket->rate); + balance = fminf( balance, bucket->burst ); + return delta <= balance; +} + FD_PROTOTYPES_END #endif /* HEADER_fd_src_waltz_fd_token_bucket_h */