From 446a6a562ebc2f0e740835fc0d2984071d0955fb Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Sun, 30 Aug 2026 16:59:12 +0000 Subject: [PATCH 01/23] failover: add the HELLO message and the session state machine --- src/discof/failover/Local.mk | 7 ++ src/discof/failover/fd_failover_proto.c | 67 +++++++++++++ src/discof/failover/fd_failover_proto.h | 114 ++++++++++++++++++++++ src/discof/failover/test_failover_proto.c | 45 +++++++++ 4 files changed, 233 insertions(+) create mode 100644 src/discof/failover/Local.mk create mode 100644 src/discof/failover/fd_failover_proto.c create mode 100644 src/discof/failover/fd_failover_proto.h create mode 100644 src/discof/failover/test_failover_proto.c diff --git a/src/discof/failover/Local.mk b/src/discof/failover/Local.mk new file mode 100644 index 00000000000..94f0ea7b55b --- /dev/null +++ b/src/discof/failover/Local.mk @@ -0,0 +1,7 @@ +$(call add-hdrs,fd_failover_proto.h) +$(call add-objs,fd_failover_proto,fd_discof) + +ifdef FD_HAS_HOSTED +$(call make-unit-test,test_failover_proto,test_failover_proto,fd_discof fd_choreo fd_flamenco fd_tango fd_ballet fd_util) +$(call run-unit-test,test_failover_proto) +endif diff --git a/src/discof/failover/fd_failover_proto.c b/src/discof/failover/fd_failover_proto.c new file mode 100644 index 00000000000..1bd710ad637 --- /dev/null +++ b/src/discof/failover/fd_failover_proto.c @@ -0,0 +1,67 @@ +#include "fd_failover_proto.h" + +int +fd_failover_hello_check( fd_failover_hello_t const * self, + fd_failover_hello_t const * peer ) { + if( FD_UNLIKELY( self->version!=peer->version ) ) return FD_FAILOVER_HELLO_ERR_VERSION; + if( FD_UNLIKELY( self->role>FD_FAILOVER_ROLE_ACTIVE || + peer->role>FD_FAILOVER_ROLE_ACTIVE ) ) return FD_FAILOVER_HELLO_ERR_ROLE; + if( FD_UNLIKELY( self->mode>=FD_FAILOVER_MODE_CNT || + self->mode!=peer->mode ) ) return FD_FAILOVER_HELLO_ERR_MODE; + if( FD_UNLIKELY( !self->boot_id || !peer->boot_id ) ) return FD_FAILOVER_HELLO_ERR_BOOT_ID; + if( FD_UNLIKELY( !fd_memeq( self->staked_pubkey, peer->staked_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_STAKED; + if( FD_UNLIKELY( !fd_memeq( self->vote_account, peer->vote_account, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_VOTE_ACCT; + if( FD_UNLIKELY( fd_memeq( self->junk_pubkey, peer->junk_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_JUNK_EQ; + if( FD_UNLIKELY( fd_memeq( self->junk_pubkey, self->staked_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_JUNK_STAKE; + if( FD_UNLIKELY( fd_memeq( peer->junk_pubkey, peer->staked_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_JUNK_STAKE; + if( FD_UNLIKELY( self->role==FD_FAILOVER_ROLE_ACTIVE && + peer->role==FD_FAILOVER_ROLE_ACTIVE ) ) return FD_FAILOVER_HELLO_ERR_BOTH_ACT; + return FD_FAILOVER_HELLO_OK; +} + +ulong +fd_failover_session_init( int dial_peer ) { + return dial_peer ? FD_FAILOVER_SESSION_BACKOFF : FD_FAILOVER_SESSION_LISTENING; +} + +ulong +fd_failover_session_step( ulong state, + int dial_peer, + int event ) { + if( FD_UNLIKELY( state>=FD_FAILOVER_SESSION_CNT ) ) return state; + if( FD_UNLIKELY( dial_peer<0 || dial_peer>1 ) ) return state; + if( FD_UNLIKELY( event<0 || event>=FD_FAILOVER_EV_CNT ) ) return state; + + int lost = event==FD_FAILOVER_EV_LINK_LOST || event==FD_FAILOVER_EV_TIMEOUT || event==FD_FAILOVER_EV_HELLO_FATAL; + + if( !dial_peer ) { + /* Candidates come and go without changing the listener's state. */ + switch( state ) { + case FD_FAILOVER_SESSION_LISTENING: + if( event==FD_FAILOVER_EV_HELLO_OK ) return FD_FAILOVER_SESSION_PAIRED; + break; + case FD_FAILOVER_SESSION_PAIRED: + if( lost ) return FD_FAILOVER_SESSION_LISTENING; + break; + } + return state; + } + + switch( state ) { + case FD_FAILOVER_SESSION_BACKOFF: + if( event==FD_FAILOVER_EV_RETRY ) return FD_FAILOVER_SESSION_DIALING; + break; + case FD_FAILOVER_SESSION_DIALING: + if( event==FD_FAILOVER_EV_CONNECTED ) return FD_FAILOVER_SESSION_HELLO; + if( lost ) return FD_FAILOVER_SESSION_BACKOFF; + break; + case FD_FAILOVER_SESSION_HELLO: + if( event==FD_FAILOVER_EV_HELLO_OK ) return FD_FAILOVER_SESSION_PAIRED; + if( lost ) return FD_FAILOVER_SESSION_BACKOFF; + break; + case FD_FAILOVER_SESSION_PAIRED: + if( lost ) return FD_FAILOVER_SESSION_BACKOFF; + break; + } + return state; +} diff --git a/src/discof/failover/fd_failover_proto.h b/src/discof/failover/fd_failover_proto.h new file mode 100644 index 00000000000..8479dc023f1 --- /dev/null +++ b/src/discof/failover/fd_failover_proto.h @@ -0,0 +1,114 @@ +#ifndef HEADER_fd_src_discof_failover_fd_failover_proto_h +#define HEADER_fd_src_discof_failover_fd_failover_proto_h + +/* Failover protocol messages and pure session transitions */ + +#include "../../util/fd_util_base.h" +#include "../../ballet/sha512/fd_sha512.h" + +/* Protocol version */ +#define FD_FAILOVER_VERSION (1U) + +/* Message types */ +#define FD_FAILOVER_MSG_HELLO (0U) +#define FD_FAILOVER_MSG_HANDOFF_REQUEST (1U) +#define FD_FAILOVER_MSG_DEMOTED (2U) +#define FD_FAILOVER_MSG_PROMOTE_ACK (3U) +#define FD_FAILOVER_MSG_PROMOTE_REJECTED (4U) +#define FD_FAILOVER_MSG_HANDOFF_RESULT (5U) +#define FD_FAILOVER_MSG_RESERVED (6U) + +/* Sentinel for a slot field with no value */ +#define FD_FAILOVER_SLOT_NULL (ULONG_MAX) + +/* Consensus payload formats */ +#define FD_FAILOVER_MODE_TOWER (0U) +#define FD_FAILOVER_MODE_CNT (1U) + +/* Roles for each endpoint */ +#define FD_FAILOVER_ROLE_STANDBY (0UL) +#define FD_FAILOVER_ROLE_ACTIVE (1UL) + +/* Session states for one authenticated TCP session per pair. The + listener only ever shows LISTENING or PAIRED, its candidates are + tracked per socket. The dialer walks BACKOFF, DIALING, HELLO and + PAIRED. */ +#define FD_FAILOVER_SESSION_LISTENING (0UL) +#define FD_FAILOVER_SESSION_DIALING (1UL) +#define FD_FAILOVER_SESSION_HELLO (2UL) +#define FD_FAILOVER_SESSION_PAIRED (3UL) +#define FD_FAILOVER_SESSION_BACKOFF (4UL) +#define FD_FAILOVER_SESSION_CNT (5UL) + +/* Session events, inputs to fd_failover_session_step */ +#define FD_FAILOVER_EV_PEER_CONNECTED (0) +#define FD_FAILOVER_EV_CONNECTED (1) +#define FD_FAILOVER_EV_HELLO_OK (2) +#define FD_FAILOVER_EV_HELLO_FATAL (3) +#define FD_FAILOVER_EV_TIMEOUT (4) +#define FD_FAILOVER_EV_LINK_LOST (5) +#define FD_FAILOVER_EV_RETRY (6) +#define FD_FAILOVER_EV_CNT (7) + +/* HELLO pairing outcomes */ +#define FD_FAILOVER_HELLO_OK (0) +#define FD_FAILOVER_HELLO_ERR_VERSION (1) +#define FD_FAILOVER_HELLO_ERR_STAKED (2) +#define FD_FAILOVER_HELLO_ERR_VOTE_ACCT (3) +#define FD_FAILOVER_HELLO_ERR_JUNK_EQ (4) +#define FD_FAILOVER_HELLO_ERR_JUNK_STAKE (5) +#define FD_FAILOVER_HELLO_ERR_BOTH_ACT (6) +#define FD_FAILOVER_HELLO_ERR_ROLE (7) +#define FD_FAILOVER_HELLO_ERR_BOOT_ID (8) +#define FD_FAILOVER_HELLO_ERR_MODE (9) +#define FD_FAILOVER_HELLO_ERR_CERT (10) + +/* Upper bound on the consensus state payload in tower mode. A + CompactTowerSync with block id and bank hash is under 512 bytes. */ +#define FD_FAILOVER_TOWER_STATE_MAX (512UL) + +/* Wire protocol message bodies. Little endian, packed, fixed layout. */ +struct __attribute__((packed)) fd_failover_hello { + ushort version; /* FD_FAILOVER_VERSION */ + uchar junk_pubkey[ 32 ]; /* this host's junk identity */ + uchar staked_pubkey[ 32 ]; /* the active identity for this pair */ + uchar vote_account[ 32 ]; /* the vote account for this pair */ + uchar role; /* FD_FAILOVER_ROLE_* role at pairing */ + uchar mode; /* FD_FAILOVER_MODE_* consensus */ + ulong boot_id; /* random nonzero value per boot to distinguish restarts */ + uchar commit[ 20 ]; /* FD commit hash */ + uchar member_cert[ 64 ]; /* staked key's signature over the cert prefix and junk_pubkey */ +}; +typedef struct fd_failover_hello fd_failover_hello_t; +FD_STATIC_ASSERT( sizeof(fd_failover_hello_t)==192UL, wire_layout ); + +FD_PROTOTYPES_BEGIN + +/* Checks a peer's HELLO against ours and returns FD_FAILOVER_HELLO_OK or + the first fatal error. The checks are symmetric, so a misconfigured + pair fails the same way on both nodes. */ +int +fd_failover_hello_check( fd_failover_hello_t const * self, + fd_failover_hello_t const * peer ); + +/* fd_failover_session_init returns the resting state of an endpoint, + LISTENING for the listener and BACKOFF for the dialer. */ +ulong +fd_failover_session_init( int dial_peer ); + +/* Returns the next session state for event on a listener or dialer. The + channel makes every state change with it or with + fd_failover_session_init. A listener stays + LISTENING while candidates handshake, pairs on HELLO_OK and goes back + to LISTENING on any loss. A dialer leaves BACKOFF on RETRY, reaches + HELLO once TCP connects, pairs on HELLO_OK and falls back to BACKOFF on + any loss. An event that does not apply, or an out of range input, + leaves the state unchanged. */ +ulong +fd_failover_session_step( ulong state, + int dial_peer, + int event ); + +FD_PROTOTYPES_END + +#endif /* HEADER_fd_src_discof_failover_fd_failover_proto_h */ diff --git a/src/discof/failover/test_failover_proto.c b/src/discof/failover/test_failover_proto.c new file mode 100644 index 00000000000..42d0348214a --- /dev/null +++ b/src/discof/failover/test_failover_proto.c @@ -0,0 +1,45 @@ +#include "fd_failover_proto.h" +#include "../../util/fd_util.h" + +static void +fill_hello( fd_failover_hello_t * h, + uchar junk, + uchar staked, + uchar vote, + uchar role ) { + fd_memset( h, 0, sizeof(fd_failover_hello_t) ); + h->version = (ushort)FD_FAILOVER_VERSION; + fd_memset( h->junk_pubkey, junk, 32UL ); + fd_memset( h->staked_pubkey, staked, 32UL ); + fd_memset( h->vote_account, vote, 32UL ); + h->role = role; + h->mode = (uchar)FD_FAILOVER_MODE_TOWER; + h->boot_id = 0x1000UL+junk; +} + +static void +test_hello( void ) { + fd_failover_hello_t active, standby; + fill_hello( &active, 1U, 0xAA, 0xBB, FD_FAILOVER_ROLE_ACTIVE ); + fill_hello( &standby, 2U, 0xAA, 0xBB, FD_FAILOVER_ROLE_STANDBY ); + FD_TEST( fd_failover_hello_check( &active, &standby )==FD_FAILOVER_HELLO_OK ); + FD_TEST( fd_failover_hello_check( &standby, &active )==FD_FAILOVER_HELLO_OK ); +} + +static void +test_wrong_staked_key( void ) { + fd_failover_hello_t active, standby; + fill_hello( &active, 1U, 0xAA, 0xBB, FD_FAILOVER_ROLE_ACTIVE ); + fill_hello( &standby, 2U, 0xAC, 0xBB, FD_FAILOVER_ROLE_STANDBY ); + FD_TEST( fd_failover_hello_check( &active, &standby )==FD_FAILOVER_HELLO_ERR_STAKED ); +} + +int +main( int argc, char ** argv ) { + fd_boot( &argc, &argv ); + test_hello(); + test_wrong_staked_key(); + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} From 2fbc1123796f81f53c8e2a2e743af432f7500f14 Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Wed, 9 Sep 2026 18:20:59 +0000 Subject: [PATCH 02/23] failover: frame the protocol messages on the wire --- src/discof/failover/Local.mk | 5 +- src/discof/failover/fd_failover_wire.c | 66 +++++++++++++++++++++ src/discof/failover/fd_failover_wire.h | 70 +++++++++++++++++++++++ src/discof/failover/test_failover_wire.c | 73 ++++++++++++++++++++++++ 4 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 src/discof/failover/fd_failover_wire.c create mode 100644 src/discof/failover/fd_failover_wire.h create mode 100644 src/discof/failover/test_failover_wire.c diff --git a/src/discof/failover/Local.mk b/src/discof/failover/Local.mk index 94f0ea7b55b..987323b173b 100644 --- a/src/discof/failover/Local.mk +++ b/src/discof/failover/Local.mk @@ -1,7 +1,10 @@ -$(call add-hdrs,fd_failover_proto.h) +$(call add-hdrs,fd_failover_proto.h fd_failover_wire.h) $(call add-objs,fd_failover_proto,fd_discof) +$(call add-objs,fd_failover_wire,fd_discof) ifdef FD_HAS_HOSTED $(call make-unit-test,test_failover_proto,test_failover_proto,fd_discof fd_choreo fd_flamenco fd_tango fd_ballet fd_util) $(call run-unit-test,test_failover_proto) +$(call make-unit-test,test_failover_wire,test_failover_wire,fd_discof fd_ballet fd_util) +$(call run-unit-test,test_failover_wire) endif diff --git a/src/discof/failover/fd_failover_wire.c b/src/discof/failover/fd_failover_wire.c new file mode 100644 index 00000000000..08b687f859f --- /dev/null +++ b/src/discof/failover/fd_failover_wire.c @@ -0,0 +1,66 @@ +#include "fd_failover_wire.h" + +#include + +void +fd_failover_wire_session_init( fd_failover_wire_session_t * session ) { + session->tx_seq = 0UL; + session->rx_seq = 0UL; +} + +ulong +fd_failover_wire_encode( fd_failover_wire_session_t * session, + uchar * out, + ushort type, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( type>=FD_FAILOVER_MSG_RESERVED || + payload_sz>FD_FAILOVER_PAYLOAD_MAX || + session->tx_seq==ULONG_MAX ) ) return 0UL; + + ulong frame_sz = FD_FAILOVER_FRAME_HDR_SZ+payload_sz; + + fd_failover_frame_hdr_t hdr = { + .version = (ushort)FD_FAILOVER_VERSION, + .len = (uint)frame_sz, + .type = type, + .seq = session->tx_seq, + }; + fd_memcpy( out, &hdr, FD_FAILOVER_FRAME_HDR_SZ ); + if( FD_LIKELY( payload_sz ) ) fd_memcpy( out+FD_FAILOVER_FRAME_HDR_SZ, payload, payload_sz ); + + session->tx_seq++; + return frame_sz; +} + +int +fd_failover_wire_decode( fd_failover_wire_session_t * session, + uchar const * buf, + ulong buf_sz, + ushort * out_type, + uchar const ** out_payload, + ulong * out_payload_sz, + ulong * out_frame_sz ) { + if( FD_UNLIKELY( buf_szFD_FAILOVER_FRAME_MAX ) ) return FD_FAILOVER_WIRE_ERR_SZ; + if( FD_UNLIKELY( buf_szrx_seq==ULONG_MAX || hdr.seq!=session->rx_seq ) ) return FD_FAILOVER_WIRE_ERR_SEQ; + if( FD_UNLIKELY( hdr.type>=FD_FAILOVER_MSG_RESERVED ) ) return FD_FAILOVER_WIRE_ERR_TYPE; + + session->rx_seq++; + *out_type = hdr.type; + *out_payload = buf+FD_FAILOVER_FRAME_HDR_SZ; + *out_payload_sz = payload_sz; + *out_frame_sz = frame_sz; + return FD_FAILOVER_WIRE_SUCCESS; +} diff --git a/src/discof/failover/fd_failover_wire.h b/src/discof/failover/fd_failover_wire.h new file mode 100644 index 00000000000..9218df6fbc4 --- /dev/null +++ b/src/discof/failover/fd_failover_wire.h @@ -0,0 +1,70 @@ +#ifndef HEADER_fd_src_discof_failover_fd_failover_wire_h +#define HEADER_fd_src_discof_failover_fd_failover_wire_h + +/* Bounded framing inside an authenticated TLS connection. This codec + does not handle authentication, it only decodes plaintext obtained + from TLS. */ + +#include "fd_failover_proto.h" + +/* Frame layout bounds */ +#define FD_FAILOVER_FRAME_HDR_SZ (16UL) +#define FD_FAILOVER_PAYLOAD_MAX (65536UL) +#define FD_FAILOVER_FRAME_MAX (FD_FAILOVER_FRAME_HDR_SZ+FD_FAILOVER_PAYLOAD_MAX) + +/* Decode results. AGAIN means the caller has not buffered the whole frame yet, and every other error is fatal to the session. */ +#define FD_FAILOVER_WIRE_SUCCESS (0) +#define FD_FAILOVER_WIRE_ERR_AGAIN (-1) +#define FD_FAILOVER_WIRE_ERR_VERSION (-2) +#define FD_FAILOVER_WIRE_ERR_SZ (-3) +#define FD_FAILOVER_WIRE_ERR_TYPE (-4) +#define FD_FAILOVER_WIRE_ERR_SEQ (-5) + +struct __attribute__((packed)) fd_failover_frame_hdr { + ushort version; /* FD_FAILOVER_VERSION */ + uint len; /* total frame length in bytes */ + ushort type; /* FD_FAILOVER_MSG_* */ + ulong seq; /* per direction, starts at zero */ +}; + +typedef struct fd_failover_frame_hdr fd_failover_frame_hdr_t; + +FD_STATIC_ASSERT( sizeof(fd_failover_frame_hdr_t)==FD_FAILOVER_FRAME_HDR_SZ, wire_layout ); + +/* Sequence counters belong to one TLS connection. */ +struct fd_failover_wire_session { + ulong tx_seq; + ulong rx_seq; +}; + +typedef struct fd_failover_wire_session fd_failover_wire_session_t; + +FD_PROTOTYPES_BEGIN + +/* Resets the sequence counters for a new TLS connection. */ +void +fd_failover_wire_session_init( fd_failover_wire_session_t * session ); + +/* Writes one frame and payload into out, which must hold at least + FD_FAILOVER_FRAME_MAX bytes. */ +ulong +fd_failover_wire_encode( fd_failover_wire_session_t * session, + uchar * out, + ushort type, + uchar const * payload, + ulong payload_sz ); + +/* Decodes the frame at the start of buf and, on success, points + *out_payload and *out_payload_sz into buf. */ +int +fd_failover_wire_decode( fd_failover_wire_session_t * session, + uchar const * buf, + ulong buf_sz, + ushort * out_type, + uchar const ** out_payload, + ulong * out_payload_sz, + ulong * out_frame_sz ); + +FD_PROTOTYPES_END + +#endif /* HEADER_fd_src_discof_failover_fd_failover_wire_h */ diff --git a/src/discof/failover/test_failover_wire.c b/src/discof/failover/test_failover_wire.c new file mode 100644 index 00000000000..0f5d557276c --- /dev/null +++ b/src/discof/failover/test_failover_wire.c @@ -0,0 +1,73 @@ +#include "fd_failover_wire.h" +#include "../../util/fd_util.h" + +static uchar frame[ FD_FAILOVER_FRAME_MAX ]; +static uchar big[ 2UL*FD_FAILOVER_FRAME_MAX ]; + +static void +paired_sessions( fd_failover_wire_session_t * a, + fd_failover_wire_session_t * b ) { + fd_failover_wire_session_init( a ); + fd_failover_wire_session_init( b ); +} + +static void +test_roundtrip( void ) { + fd_failover_wire_session_t a, b; + paired_sessions( &a, &b ); + + uchar payload[ 70 ]; + for( ulong i=0UL; i<70UL; i++ ) payload[ i ] = (uchar)i; + + ulong sz = fd_failover_wire_encode( &a, frame, (ushort)FD_FAILOVER_MSG_DEMOTED, payload, 70UL ); + FD_TEST( sz==FD_FAILOVER_FRAME_HDR_SZ+70UL ); + + ushort type; + uchar const * out; + ulong out_sz; + ulong frame_sz; + FD_TEST( fd_failover_wire_decode( &b, frame, sz, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); + FD_TEST( type==(ushort)FD_FAILOVER_MSG_DEMOTED ); + FD_TEST( out_sz==70UL ); + FD_TEST( frame_sz==sz ); + FD_TEST( fd_memeq( out, payload, 70UL ) ); + + ulong sz2 = fd_failover_wire_encode( &a, big, (ushort)FD_FAILOVER_MSG_HELLO, payload, 10UL ); + fd_memset( big+sz2, 0x77, 64UL ); + FD_TEST( fd_failover_wire_decode( &b, big, sz2+64UL, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); + FD_TEST( frame_sz==sz2 ); + + FD_LOG_NOTICE(( "pass: test_roundtrip" )); +} + +static void +test_replay( void ) { + fd_failover_wire_session_t a, b; + paired_sessions( &a, &b ); + + uchar payload[ 27 ] = { 5 }; + ulong sz = fd_failover_wire_encode( &a, frame, (ushort)FD_FAILOVER_MSG_DEMOTED, payload, 27UL ); + + ushort type; + uchar const * out; + ulong out_sz; + ulong frame_sz; + + FD_TEST( fd_failover_wire_decode( &b, frame, sz, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); + FD_TEST( fd_failover_wire_decode( &b, frame, sz, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_ERR_SEQ ); + + FD_LOG_NOTICE(( "pass: test_replay" )); +} + +int +main( int argc, + char ** argv ) { + fd_boot( &argc, &argv ); + + test_roundtrip(); + test_replay(); + + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} From 267cffc0517cce302c404a1edf3cc4338b232626 Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Wed, 9 Sep 2026 18:29:38 +0000 Subject: [PATCH 03/23] failover: add the [failover] config and derive the junk identity path --- src/app/firedancer/config/default.toml | 31 +++++++++++++++ src/app/shared/fd_config.c | 38 ++++++++++++++++++ src/app/shared/fd_config.h | 7 ++++ src/app/shared/fd_config_json.c | 9 +++++ src/app/shared/fd_config_parse.c | 4 ++ src/app/shared/test_config_json.c | 10 +++++ src/app/shared/test_config_parse.c | 53 +++++++++++++++++++++++++- 7 files changed, 151 insertions(+), 1 deletion(-) diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index 32c2bac2711..e486e15ce29 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -1974,6 +1974,37 @@ telemetry = true # loading. delay_startup = true +# Firedancer validators support a special failover protocol between one +# another, which guarantees fast, safe, at-most-once handover with no +# equivocation or voting violations. +# +# The protocol works by hosting a TCP listener on your validator. Any +# peer in possession of the currently active identity private key can +# demote this validator to the junk identity by sending a specially +# signed packet proving that possession. +# +# Between two Firedancer validators, only hand off the identity with +# `failover promote`. Do not use `set-identity --vote-history-file` +# for this. The failover protocol passes the final vote history +# directly and checks it against what this validator replayed. Vote +# history files are only for handing off an identity from a validator that +# does not run failover, such as Agave. +[failover] + # Whether to enable the failover protocol on this validator, which + # exposes the TCP listener and allows peers to securely demote it to + # the junk identity. + enabled = false + + # The address to listen on. If enabled, by default the listener is + # exposed to the network. This is safe as possession of the private + # key is required to demote the node, but could be a vector for + # a targeted denial of service. For added reliability, restrict + # the listener to a private network. + listen_address = "0.0.0.0" + + # The port to listen on for handoff requests. + listen_port = 8010 + # These options can be useful for development, but should not be used # when connecting to a live cluster, as they may cause the validator to # be unstable or have degraded performance or security. The program diff --git a/src/app/shared/fd_config.c b/src/app/shared/fd_config.c index 12d3eaad4c6..042360b9632 100644 --- a/src/app/shared/fd_config.c +++ b/src/app/shared/fd_config.c @@ -181,6 +181,14 @@ fd_config_fillf( fd_config_t * config ) { replace( config->firedancer.paths.authorized_voter_paths[ i ], "{user}", config->user ); replace( config->firedancer.paths.authorized_voter_paths[ i ], "{name}", config->name ); } + + if( FD_UNLIKELY( config->firedancer.failover.enabled ) ) { + FD_TEST( fd_cstr_printf_check( config->firedancer.failover.junk_identity_path, + sizeof(config->firedancer.failover.junk_identity_path), + NULL, + "%s/junk-identity.json", + config->paths.base ) ); + } } static void @@ -343,6 +351,12 @@ fd_config_fill( fd_config_t * config, replace( config->paths.base, "{user}", config->user ); replace( config->paths.base, "{name}", config->name ); + /* Under failover [paths.identity_key] is the staked key both machines + share. */ + if( FD_UNLIKELY( config->firedancer.failover.enabled && !strcmp( config->paths.identity_key, "" ) ) ) { + FD_LOG_ERR(( "[failover.enabled] is true, so [paths.identity_key] must specify the staked identity keypair both machines share" )); + } + if( FD_UNLIKELY( !strcmp( config->paths.identity_key, "" ) ) ) { /* Development binaries generate an identity key on boot. */ if( FD_UNLIKELY( config->is_live_cluster && !dev ) ) FD_LOG_ERR(( "configuration file must specify [paths.identity_key] when joining a live cluster" )); @@ -519,9 +533,33 @@ fd_config_validatef( fd_configf_t const * config ) { CFG_HAS_NON_ZERO( runtime.program_cache_size_mib ); } +static void +fd_config_validate_failover( fd_config_t const * config ) { + fd_configf_t const * f = &config->firedancer; + + if( FD_LIKELY( !f->failover.enabled ) ) return; + if( FD_UNLIKELY( !f->failover.listen_port ) ) { + FD_LOG_ERR(( "[failover.listen_port] must not be zero" )); + } + uint listen_addr; + if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( f->failover.listen_address, &listen_addr ) ) ) { + FD_LOG_ERR(( "[failover.listen_address] `%s` must be an IPv4 address, host names are not accepted", f->failover.listen_address )); + } + if( FD_UNLIKELY( !strcmp( config->paths.vote_account, "" ) ) ) { + FD_LOG_ERR(( "[failover.enabled] is true, so [paths.vote_account] must be set to the vote account both machines vote for" )); + } + /* Failover is built on the Tower tiles, and Votor does not adopt a + vote history or follow the staked key's BLS key through a switch. */ + if( FD_UNLIKELY( f->consensus.alpenglow ) ) { + FD_LOG_ERR(( "failover in this build supports Tower only, set [consensus.alpenglow] to false or [failover.enabled] to false" )); + } +} + + void fd_config_validate( fd_config_t const * config ) { fd_config_validatef( &config->firedancer ); + fd_config_validate_failover( config ); CFG_HAS_NON_EMPTY( name ); CFG_HAS_NON_EMPTY( paths.base ); diff --git a/src/app/shared/fd_config.h b/src/app/shared/fd_config.h index 86a4126188f..0f20870a307 100644 --- a/src/app/shared/fd_config.h +++ b/src/app/shared/fd_config.h @@ -126,6 +126,13 @@ struct fd_configf { char authorized_voter_paths[ FD_KEYGUARD_AUTH_VOTERS_MAX ][ PATH_MAX ]; } paths; + struct { + int enabled; + char listen_address[ 16 ]; + ushort listen_port; + char junk_identity_path[ PATH_MAX ]; /* derived, [paths.base]/junk-identity.json when enabled */ + } failover; + }; typedef struct fd_configf fd_configf_t; diff --git a/src/app/shared/fd_config_json.c b/src/app/shared/fd_config_json.c index 56129762d1a..dc2e9f3bcf6 100644 --- a/src/app/shared/fd_config_json.c +++ b/src/app/shared/fd_config_json.c @@ -126,6 +126,7 @@ static char const * const jw_redacted_keys[] = { "snapshots.sources.servers", "snapshots.server.http_listen_address", "hugetlbfs.mount_path", + "failover.listen_address", "net.bind_address", "tiles.quic.ssl_key_log_file", "tiles.bundle.url", @@ -419,6 +420,14 @@ fd_config_to_json( fd_config_t const * config, jw_obj_close( &w ); jw_obj_close( &w ); + if( FD_UNLIKELY( f->failover.enabled ) ) { + jw_obj_open( &w, "failover" ); + jw_bool ( &w, "enabled", f->failover.enabled ); + jw_path ( &w, "listen_address", f->failover.listen_address ); + jw_ulong( &w, "listen_port", f->failover.listen_port ); + jw_obj_close( &w ); + } + jw_obj_open( &w, "hugetlbfs" ); jw_path ( &w, "gigantic_page_mount_path", config->hugetlbfs.gigantic_page_mount_path ); jw_path ( &w, "huge_page_mount_path", config->hugetlbfs.huge_page_mount_path ); diff --git a/src/app/shared/fd_config_parse.c b/src/app/shared/fd_config_parse.c index ad342d3494a..cf9c7cea8ef 100644 --- a/src/app/shared/fd_config_parse.c +++ b/src/app/shared/fd_config_parse.c @@ -77,6 +77,10 @@ fd_config_extract_podf( uchar * pod, CFG_POP ( ulong, snapshots.server.send_timeout_millis ); CFG_POP ( ulong, snapshots.server.send_buffer_size_kib ); + CFG_POP ( bool, failover.enabled ); + CFG_POP ( cstr, failover.listen_address ); + CFG_POP ( ushort, failover.listen_port ); + CFG_POP ( bool, development.hard_fork_fatal ); CFG_POP ( bool, development.fixed_fec_sets ); diff --git a/src/app/shared/test_config_json.c b/src/app/shared/test_config_json.c index a38d143381a..92db78ef362 100644 --- a/src/app/shared/test_config_json.c +++ b/src/app/shared/test_config_json.c @@ -139,6 +139,16 @@ main( int argc, FD_TEST( !strstr( json, "hunter2" ) ); FD_TEST( !strstr( json, "example.com" ) ); + /* without failover nothing about it is reported */ + FD_TEST( !strstr( json, "\"failover\"" ) ); + + /* the listen address is redacted, the rest is reported */ + config->firedancer.failover.enabled = 1; + strcpy( config->firedancer.failover.listen_address, "10.9.8.7" ); + FD_TEST( fd_config_to_json( config, json, sizeof(json) ) ); + FD_TEST( !strstr( json, "10.9.8.7" ) ); + FD_TEST( strstr( json, "\"failover\":{\"enabled\":true,\"listen_address\":\"[redacted]\",\"listen_port\":8010}" ) ); + /* user override toml renders with keyword redaction */ static char const user_toml[] = "name = \"fd1\"\n" diff --git a/src/app/shared/test_config_parse.c b/src/app/shared/test_config_parse.c index ccbf91fa08a..0e8ef7222ad 100644 --- a/src/app/shared/test_config_parse.c +++ b/src/app/shared/test_config_parse.c @@ -31,6 +31,11 @@ static char const cfg_str_5[] = static char const cfg_str_vote_history[] = "[paths]\n" " vote_history = \"/data/{name}/vote_history\"\n"; +static char const cfg_str_failover[] = + "[failover]\n" + " enabled = true\n" + " listen_address = \"10.0.0.1\"\n" + " listen_port = 9010"; extern uchar const firedancer_default_config[]; extern ulong const firedancer_default_config_sz; @@ -71,7 +76,6 @@ vote_history_path_is_valid( char const * path ) { return WIFEXITED( status ) && !WEXITSTATUS( status ); } - /* Runs fd_config_validate_consensus on the given combination and reports whether it was accepted. It exits the process on rejection, so run it in a child. */ @@ -99,6 +103,34 @@ consensus_is_valid( ulong wfs_slot, return WIFEXITED( status ) && !WEXITSTATUS( status ); } +/* Validates the config with the given failover knobs in a child, so the + FD_LOG_ERR paths can be checked. */ + +static int +failover_is_valid( config_t * config, + int enabled, + char const * listen_address, + ushort listen_port, + char const * vote_account, + int alpenglow ) { + int pid = fork(); + FD_TEST( pid>=0 ); + if( FD_UNLIKELY( !pid ) ) { + config->firedancer.development.genesis.max_file_size_mib = 4055UL; + config->firedancer.consensus.alpenglow = alpenglow; + config->firedancer.failover.enabled = enabled; + config->firedancer.failover.listen_port = listen_port; + fd_cstr_ncpy( config->firedancer.failover.listen_address, listen_address, sizeof(config->firedancer.failover.listen_address) ); + fd_cstr_ncpy( config->paths.vote_account, vote_account, sizeof(config->paths.vote_account) ); + fd_config_validate( config ); + _exit( 0 ); + } + + int status = 0; + FD_TEST( waitpid( pid, &status, 0 )==pid ); + return WIFEXITED( status ) && !WEXITSTATUS( status ); +} + int main( int argc, char ** argv ) { @@ -204,6 +236,14 @@ main( int argc, FD_TEST( genesis_max_file_size_is_valid( config, 4055UL ) ); FD_TEST( !genesis_max_file_size_is_valid( config, 4056UL ) ); + FD_TEST( failover_is_valid( config, 1, "0.0.0.0", 8010, "vote.json", 0 ) ); + FD_TEST( failover_is_valid( config, 1, "10.0.0.1", 8010, "vote.json", 0 ) ); + FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 0, "vote.json", 0 ) ); + FD_TEST( !failover_is_valid( config, 1, "localhost", 8010, "vote.json", 0 ) ); + FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 8010, "", 0 ) ); + FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 8010, "vote.json", 1 ) ); + FD_TEST( failover_is_valid( config, 0, "", 0, "", 1 ) ); + /* Ensure we can selectively override a field */ config->gossip.port = 9191; @@ -280,6 +320,17 @@ main( int argc, # undef HASH + /* Parse the failover knobs */ + + fd_memset( config, 0, sizeof(config_t) ); + config->is_firedancer = 1; + pod = fd_pod_join( fd_pod_new( pod_mem, sizeof(pod_mem) ) ); + FD_TEST( fd_toml_parse( cfg_str_failover, sizeof(cfg_str_failover)-1, pod, scratch, sizeof(scratch), NULL ) == FD_TOML_SUCCESS ); + FD_TEST( fd_config_extract_pod( pod, config ) == config ); + FD_TEST( config->firedancer.failover.enabled ); + FD_TEST( !strcmp( config->firedancer.failover.listen_address, "10.0.0.1" ) ); + FD_TEST( config->firedancer.failover.listen_port==9010 ); + FD_LOG_NOTICE(( "pass" )); fd_halt(); } From ee99d9279e7df3a9aa9b50966aec639f4686c694 Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Fri, 18 Sep 2026 16:58:16 +0000 Subject: [PATCH 04/23] failover: generate the junk identity and boot every tile under it --- src/app/firedancer/topology.c | 38 +++++++------ src/app/shared/commands/run/run.c | 88 +++++++++++++++++++++++++++++++ 2 files changed, 111 insertions(+), 15 deletions(-) diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index 1ddd15f83ec..3b596b793e7 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -1368,6 +1368,14 @@ wfs_decode_bank_hash( fd_config_t const * config, void fd_topo_configure_tile( fd_topo_tile_t * tile, fd_config_t * config ) { + /* Failover machines boot under the junk identity, only promotion + installs the staked key. A demoted machine rebooting under the + staked key would make the doppelganger rule shut down the promoted + peer instead. */ + char const * identity_key_path = config->firedancer.failover.enabled + ? config->firedancer.failover.junk_identity_path + : config->paths.identity_key; + if( FD_UNLIKELY( !strcmp( tile->name, "metric" ) ) ) { if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->tiles.metric.prometheus_listen_address, &tile->metric.prometheus_listen_addr ) ) ) @@ -1376,7 +1384,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "event" ) ) ) { - fd_cstr_ncpy( tile->event.identity_key_path, config->paths.identity_key, sizeof(tile->event.identity_key_path) ); + fd_cstr_ncpy( tile->event.identity_key_path, identity_key_path, sizeof(tile->event.identity_key_path) ); fd_cstr_ncpy( tile->event.url, config->tiles.event.url, sizeof(tile->event.url) ); fd_cstr_ncpy( tile->event.action, config->action, sizeof(tile->event.action) ); FD_TEST( fd_cstr_printf_check( tile->event.accounts_path, sizeof(tile->event.accounts_path), NULL, "%s", config->paths.accounts ) ); @@ -1439,11 +1447,11 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "admin" ) ) ) { - fd_cstr_ncpy( tile->admin.identity_key_path, config->paths.identity_key, sizeof(tile->admin.identity_key_path) ); + fd_cstr_ncpy( tile->admin.identity_key_path, identity_key_path, sizeof(tile->admin.identity_key_path) ); } else if( FD_UNLIKELY( !strcmp( tile->name, "gossvf") ) ) { - fd_cstr_ncpy( tile->gossvf.identity_key_path, config->paths.identity_key, sizeof(tile->gossvf.identity_key_path) ); + fd_cstr_ncpy( tile->gossvf.identity_key_path, identity_key_path, sizeof(tile->gossvf.identity_key_path) ); tile->gossvf.tcache_depth = 1<<22UL; /* TODO: user defined option */ tile->gossvf.shred_version = 0U; tile->gossvf.allow_private_address = config->development.gossip.allow_private_address; @@ -1467,7 +1475,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->gossip.net_ip_addr = config->net.ip_addr; tile->gossip.ip_addr = config->net.ip_addr; tile->gossip.bind_ip_addr = config->net.ip_addr; - fd_cstr_ncpy( tile->gossip.identity_key_path, config->paths.identity_key, sizeof(tile->gossip.identity_key_path) ); + fd_cstr_ncpy( tile->gossip.identity_key_path, identity_key_path, sizeof(tile->gossip.identity_key_path) ); tile->gossip.expected_shred_version = config->consensus.expected_shred_version; tile->gossip.max_entries = config->tiles.gossip.max_entries; tile->gossip.boot_timestamp_nanos = config->boot_timestamp_nanos; @@ -1562,13 +1570,13 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } } tile->rotor.repair_sign_cnt = config->firedancer.layout.sign_tile_count - 1; /* -1 because this excludes the keyguard client */ - fd_cstr_ncpy( tile->rotor.identity_key_path, config->paths.identity_key, sizeof(tile->rotor.identity_key_path) ); + fd_cstr_ncpy( tile->rotor.identity_key_path, identity_key_path, sizeof(tile->rotor.identity_key_path) ); } else if( FD_UNLIKELY( !strcmp( tile->name, "rserve" ) ) ) { tile->rserve.repair_serve_listen_port = config->tiles.rserve.repair_serve_listen_port; tile->rserve.max_shreds_per_block = config->limits.max_shreds_per_block; tile->rserve.ping_cache_entries = 1UL<<16; /* TODO: Configure this from some global metric? */ tile->rserve.blockdb_max = 1UL<<15; /* ~20.6 KiB each. TODO: make configurable */ - fd_cstr_ncpy( tile->rserve.identity_key_path, config->paths.identity_key, sizeof(tile->rserve.identity_key_path) ); + fd_cstr_ncpy( tile->rserve.identity_key_path, identity_key_path, sizeof(tile->rserve.identity_key_path) ); } else if( FD_UNLIKELY( !strcmp( tile->name, "replay" ) )) { @@ -1584,7 +1592,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->replay.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX ); FD_TEST( tile->replay.txncache_obj_id !=ULONG_MAX ); - fd_cstr_ncpy( tile->replay.identity_key_path, config->paths.identity_key, sizeof(tile->replay.identity_key_path) ); + fd_cstr_ncpy( tile->replay.identity_key_path, identity_key_path, sizeof(tile->replay.identity_key_path) ); tile->replay.ip_addr = config->net.ip_addr; fd_cstr_ncpy( tile->replay.vote_account_path, config->paths.vote_account, sizeof(tile->replay.vote_account_path) ); @@ -1649,7 +1657,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->votor.quic_server_listen_port = config->firedancer.development.votor.quic_server_listen_port; tile->votor.ip_addr = config->net.ip_addr; tile->votor.max_live_slots = config->firedancer.runtime.max_live_slots; - fd_cstr_ncpy( tile->votor.identity_key_path, config->paths.identity_key, sizeof(tile->votor.identity_key_path) ); + fd_cstr_ncpy( tile->votor.identity_key_path, identity_key_path, sizeof(tile->votor.identity_key_path) ); fd_cstr_ncpy( tile->votor.vote_history_path, config->paths.vote_history, sizeof(tile->votor.vote_history_path) ); tile->votor.target_gid = config->gid; tile->votor.target_uid = config->uid; @@ -1734,7 +1742,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, PARSE_BUNDLE_PUBKEY( pack, tip_payment_program_addr ); PARSE_BUNDLE_PUBKEY( pack, tip_distribution_authority ); tile->pack.bundle.commission_bps = config->tiles.bundle.commission_bps; - fd_cstr_ncpy( tile->pack.bundle.identity_key_path, config->paths.identity_key, sizeof(tile->pack.bundle.identity_key_path) ); + fd_cstr_ncpy( tile->pack.bundle.identity_key_path, identity_key_path, sizeof(tile->pack.bundle.identity_key_path) ); fd_cstr_ncpy( tile->pack.bundle.vote_account_path, config->paths.vote_account, sizeof(tile->pack.bundle.vote_account_path) ); } else { fd_memset( &tile->pack.bundle, '\0', sizeof(tile->pack.bundle) ); @@ -1749,14 +1757,14 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->execle.report_runtime_diffs = config->development.event.report_runtime_diffs; } else if( FD_UNLIKELY( !strcmp( tile->name, "motor" ) ) ) { - fd_cstr_ncpy( tile->motor.identity_key_path, config->paths.identity_key, sizeof(tile->motor.identity_key_path) ); + fd_cstr_ncpy( tile->motor.identity_key_path, identity_key_path, sizeof(tile->motor.identity_key_path) ); tile->motor.execle_cnt = config->firedancer.layout.execle_tile_count; tile->motor.max_txn_per_slot = config->limits.max_txn_per_slot; } else if( FD_UNLIKELY( !strcmp( tile->name, "shred" ) ) ) { - fd_cstr_ncpy( tile->shred.identity_key_path, config->paths.identity_key, sizeof(tile->shred.identity_key_path) ); + fd_cstr_ncpy( tile->shred.identity_key_path, identity_key_path, sizeof(tile->shred.identity_key_path) ); tile->shred.fec_exposure = FD_SHRED_FIREDANCER_FEC_EXPOSURE; tile->shred.fec_resolver_depth = config->tiles.shred.max_pending_shred_sets; @@ -1774,7 +1782,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) ) ) { - fd_cstr_ncpy( tile->sign.identity_key_path, config->paths.identity_key, sizeof(tile->sign.identity_key_path) ); + fd_cstr_ncpy( tile->sign.identity_key_path, identity_key_path, sizeof(tile->sign.identity_key_path) ); tile->sign.authorized_voter_paths_cnt = config->firedancer.paths.authorized_voter_paths_cnt; for( ulong i=0UL; isign.authorized_voter_paths_cnt; i++ ) { @@ -1811,7 +1819,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->gui.max_txn_per_slot = config->limits.max_txn_per_slot; tile->gui.is_voting = strcmp( config->paths.vote_account, "" ); fd_cstr_ncpy( tile->gui.cluster, config->cluster, sizeof(tile->gui.cluster) ); - fd_cstr_ncpy( tile->gui.identity_key_path, config->paths.identity_key, sizeof(tile->gui.identity_key_path) ); + fd_cstr_ncpy( tile->gui.identity_key_path, identity_key_path, sizeof(tile->gui.identity_key_path) ); fd_cstr_ncpy( tile->gui.vote_key_path, config->paths.vote_account, sizeof(tile->gui.vote_key_path) ); fd_cstr_ncpy( tile->gui.accounts_database_path, config->paths.accounts, sizeof(tile->gui.accounts_database_path) ); fd_cstr_ncpy( tile->gui.gui_database_path, config->paths.guidb, sizeof(tile->gui.gui_database_path) ); @@ -1846,7 +1854,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->rpc.accdb_epoch_fseq_obj_id = fd_pod_query_ulong( config->topo.props, "accdb_epoch.rpc", ULONG_MAX ); FD_TEST( tile->rpc.accdb_epoch_fseq_obj_id!=ULONG_MAX ); - fd_cstr_ncpy( tile->rpc.identity_key_path, config->paths.identity_key, sizeof(tile->rpc.identity_key_path) ); + fd_cstr_ncpy( tile->rpc.identity_key_path, identity_key_path, sizeof(tile->rpc.identity_key_path) ); tile->rpc.snapshot_server_enabled = fd_topo_find_tile( &config->topo, "snapsv", 0UL )!=ULONG_MAX; if( FD_LIKELY( tile->rpc.snapshot_server_enabled ) ) { @@ -1872,7 +1880,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->bundle.url_len = strnlen( tile->bundle.url, sizeof(tile->bundle.url)-1UL ); fd_cstr_ncpy( tile->bundle.sni, config->tiles.bundle.tls_domain_name, sizeof(tile->bundle.sni) ); tile->bundle.sni_len = strnlen( tile->bundle.sni, sizeof(tile->bundle.sni)-1UL ); - fd_cstr_ncpy( tile->bundle.identity_key_path, config->paths.identity_key, sizeof(tile->bundle.identity_key_path) ); + fd_cstr_ncpy( tile->bundle.identity_key_path, identity_key_path, sizeof(tile->bundle.identity_key_path) ); fd_cstr_ncpy( tile->bundle.key_log_path, config->development.bundle.ssl_key_log_file, sizeof(tile->bundle.key_log_path) ); tile->bundle.buf_sz = config->development.bundle.buffer_size_kib<<10; tile->bundle.out_depth = config->tiles.verify.receive_buffer_size; diff --git a/src/app/shared/commands/run/run.c b/src/app/shared/commands/run/run.c index 24b57b8e34f..8f9af899632 100644 --- a/src/app/shared/commands/run/run.c +++ b/src/app/shared/commands/run/run.c @@ -27,6 +27,7 @@ #include "../../../../disco/sleep/fd_sleep.h" #include "../../../../util/pod/fd_pod_format.h" #include "../../../../discof/restore/utils/fd_wfs.h" +#include "../../../../ballet/ed25519/fd_ed25519.h" #include "../configure/configure.h" #include "../configure/fd_cpu_isolation.h" @@ -41,6 +42,7 @@ #include #include #include +#include #include #include #include @@ -1066,6 +1068,88 @@ fdctl_check_configure( config_t const * config ) { fd_cpu_isolation_check_cgroup( config ); } +/* Writes a new keypair to tmp and syncs it. */ +static void FD_FN_SENSITIVE +failover_junk_write( char const * tmp ) { + uchar keypair[ 64 ]; + for( long got=0L; got<32L; ) { + long n = getrandom( keypair+got, (ulong)(32L-got), GRND_RANDOM ); + if( FD_UNLIKELY( -1==n ) ) FD_LOG_ERR(( "could not create the failover junk identity, getrandom() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + got += n; + } + fd_sha512_t sha[ 1 ]; + fd_ed25519_public_from_private( keypair+32UL, keypair, fd_sha512_join( fd_sha512_new( sha ) ) ); + + /* The JSON array of the 64 bytes, as `keys new` writes it. */ + char json[ 1UL+64UL*4UL+1UL ]; + ulong len = 0UL; + for( ulong i=0UL; i<64UL; i++ ) { + ulong n; + FD_TEST( fd_cstr_printf_check( json+len, sizeof(json)-len, &n, "%s%u", i ? "," : "[", (uint)keypair[ i ] ) ); + len += n; + } + json[ len++ ] = ']'; + fd_memzero_explicit( keypair, sizeof(keypair) ); + + int fd = open( tmp, O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC, S_IRUSR|S_IWUSR ); + if( FD_UNLIKELY( -1==fd ) ) FD_LOG_ERR(( "could not create `%s` (%i-%s)", tmp, errno, fd_io_strerror( errno ) )); + long written = write( fd, json, len ); + fd_memzero_explicit( json, sizeof(json) ); + if( FD_UNLIKELY( written!=(long)len ) ) FD_LOG_ERR(( "could not write `%s` (%i-%s)", tmp, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( fsync( fd ) ) ) FD_LOG_ERR(( "could not sync `%s` (%i-%s)", tmp, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "could not close `%s` (%i-%s)", tmp, errno, fd_io_strerror( errno ) )); +} + +/* The failover junk identity is this machine's own key, kept as the + validator user in [paths.base]. */ +static void +failover_junk_prepare( config_t const * config ) { + char const * path = config->firedancer.failover.junk_identity_path; + char tmp[ PATH_MAX ]; + char dir[ PATH_MAX ]; + if( FD_UNLIKELY( !fd_cstr_printf_check( tmp, sizeof(tmp), NULL, "%s.tmp", path ) ) ) FD_LOG_ERR(( "failover junk identity path `%s` is too long", path )); + fd_cstr_ncpy( dir, path, sizeof(dir) ); + char * slash = strrchr( dir, '/' ); + if( FD_UNLIKELY( !slash ) ) FD_LOG_ERR(( "failover junk identity path `%s` has no directory", path )); + *slash = '\0'; + + gid_t gid = getegid(); + uid_t uid = geteuid(); + if( FD_LIKELY( !gid && setegid( config->gid ) ) ) FD_LOG_ERR(( "setegid() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( FD_LIKELY( !uid && seteuid( config->uid ) ) ) FD_LOG_ERR(( "seteuid() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + + /* A crash can leave the temporary copy, a whole keypair, behind. */ + if( FD_UNLIKELY( unlink( tmp ) && errno!=ENOENT ) ) FD_LOG_ERR(( "could not remove `%s` (%i-%s)", tmp, errno, fd_io_strerror( errno ) )); + + int fd = open( path, O_RDONLY|O_NOFOLLOW|O_NONBLOCK|O_CLOEXEC ); + if( FD_UNLIKELY( -1==fd && errno==ENOENT ) ) { + if( FD_UNLIKELY( -1==fd_file_util_mkdir_all( dir, config->uid, config->gid, 1 ) ) ) FD_LOG_ERR(( "could not create `%s` (%i-%s)", dir, errno, fd_io_strerror( errno ) )); + failover_junk_write( tmp ); + if( FD_UNLIKELY( rename( tmp, path ) ) ) FD_LOG_ERR(( "could not rename `%s` to `%s` (%i-%s)", tmp, path, errno, fd_io_strerror( errno ) )); + int dir_fd = open( dir, O_RDONLY|O_DIRECTORY|O_CLOEXEC ); + if( FD_UNLIKELY( -1==dir_fd || fsync( dir_fd ) || close( dir_fd ) ) ) FD_LOG_ERR(( "could not sync `%s` (%i-%s)", dir, errno, fd_io_strerror( errno ) )); + FD_LOG_NOTICE(( "created the failover junk identity `%s`, it is this machine's own unique key, never copy it", path )); + } else { + struct stat st; + if( FD_UNLIKELY( -1==fd ) ) FD_LOG_ERR(( "could not open the failover junk identity `%s` (%i-%s), it has to be a regular file", path, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( fstat( fd, &st ) ) ) FD_LOG_ERR(( "could not stat the failover junk identity `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "could not close the failover junk identity `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) )); + /* Use the same size bounds as the key loader. */ + if( FD_UNLIKELY( !S_ISREG( st.st_mode ) || st.st_size<129L || st.st_size>1023L ) ) { + FD_LOG_ERR(( "the failover junk identity `%s` is not a keypair file, it is generated, so delete it and restart to make a new one, never put the staked key there", path )); + } + if( FD_UNLIKELY( st.st_uid!=config->uid && st.st_uid ) ) { + FD_LOG_ERR(( "the failover junk identity `%s` is owned by uid %u, not by the validator user or root", path, (uint)st.st_uid )); + } + if( FD_UNLIKELY( st.st_mode & (S_IRWXG|S_IRWXO) ) ) { + FD_LOG_ERR(( "the failover junk identity `%s` can be read by other users, run `chmod 600` on it", path )); + } + } + + if( FD_UNLIKELY( seteuid( uid ) ) ) FD_LOG_ERR(( "seteuid() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( setegid( gid ) ) ) FD_LOG_ERR(( "setegid() failed (%i-%s)", errno, fd_io_strerror( errno ) )); +} + void run_firedancer_init( config_t * config, int init_workspaces, @@ -1075,6 +1159,10 @@ run_firedancer_init( config_t * config, if( FD_UNLIKELY( -1==err && errno==ENOENT ) ) FD_LOG_ERR(( "[consensus.identity_path] key does not exist `%s`. You can generate an identity key at this path by running `%s keys new %s --config `", config->paths.identity_key, FD_BINARY_NAME, config->paths.identity_key )); else if( FD_UNLIKELY( -1==err ) ) FD_LOG_ERR(( "could not stat [consensus.identity_path] `%s` (%i-%s)", config->paths.identity_key, errno, fd_io_strerror( errno ) )); + /* Failover boots under a junk identity of this machine's own, created + on the first boot. */ + if( FD_UNLIKELY( config->firedancer.failover.enabled ) ) failover_junk_prepare( config ); + /* FIXME: fdctl_check_configure unconditionally checks for network stack prerequisites even if the command being run does not require networking. Hack around that here for now. */ From 5bdb3cb433f1152ab070a75b057356d4cdfd147c Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Wed, 23 Sep 2026 20:24:27 +0000 Subject: [PATCH 05/23] failover: preload both identities and sign the member cert and pair TLS --- src/app/firedancer/commands/set_identity.c | 3 + src/app/firedancer/topology.c | 2 + src/disco/keyguard/fd_keyguard.h | 15 +- src/disco/keyguard/fd_keyguard_authorize.c | 13 ++ src/disco/keyguard/fd_keyguard_match.c | 15 ++ src/disco/keyguard/fd_keyswitch.h | 6 + src/disco/keyguard/fd_sign_tile.c | 120 +++++++++++++- src/disco/keyguard/fuzz_keyguard.c | 13 ++ src/disco/keyguard/test_keyguard.c | 60 +++++++ src/disco/keyguard/test_sign_tile.c | 184 +++++++++++++++++++++ src/disco/topo/fd_topo.c | 5 + src/disco/topo/fd_topo.h | 2 + src/discof/admin/fd_admin_tile.c | 13 ++ src/discof/failover/fd_failover_proto.c | 25 +++ src/discof/failover/fd_failover_proto.h | 14 ++ 15 files changed, 484 insertions(+), 6 deletions(-) diff --git a/src/app/firedancer/commands/set_identity.c b/src/app/firedancer/commands/set_identity.c index 6887dd660fb..ebcdf78c316 100644 --- a/src/app/firedancer/commands/set_identity.c +++ b/src/app/firedancer/commands/set_identity.c @@ -132,6 +132,9 @@ set_identity( args_t * args, case FD_SET_IDENTITY_RESULT_INVALID_VOTE_HISTORY: FD_LOG_ERR(( "Failed to set identity: the vote history file is invalid or does not belong to the new identity. " "If you believe the vote history file is valid, please contact the Firedancer team for additional assistance." )); + case FD_ADMINCTL_RESULT_UNSUPPORTED: + FD_LOG_ERR(( "Failed to set identity: the validator runs with [failover.enabled] set to true, " + "and failover fully manages the identity while it is on." )); case FD_ADMINCTL_RESULT_UNKNOWN_COMMAND: case FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH: case FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH: diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index 3b596b793e7..768ca1107c0 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -1783,6 +1783,8 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) ) ) { fd_cstr_ncpy( tile->sign.identity_key_path, identity_key_path, sizeof(tile->sign.identity_key_path) ); + tile->sign.failover_enabled = config->firedancer.failover.enabled; + fd_cstr_ncpy( tile->sign.failover_staked_key_path, config->paths.identity_key, sizeof(tile->sign.failover_staked_key_path) ); tile->sign.authorized_voter_paths_cnt = config->firedancer.paths.authorized_voter_paths_cnt; for( ulong i=0UL; isign.authorized_voter_paths_cnt; i++ ) { diff --git a/src/disco/keyguard/fd_keyguard.h b/src/disco/keyguard/fd_keyguard.h index b852636d91f..161afd2c3f2 100644 --- a/src/disco/keyguard/fd_keyguard.h +++ b/src/disco/keyguard/fd_keyguard.h @@ -36,7 +36,18 @@ FD_STATIC_ASSERT( FD_KEYGUARD_AUTH_VOTERS_MAX<=16UL, auth_voters_max_fits_sign_r #define FD_KEYGUARD_ROLE_RSERVE (8) /* Repair server tile */ #define FD_KEYGUARD_ROLE_VOTOR (9) /* Alpenglow votor tile (QUIC TLS, vote history file) */ #define FD_KEYGUARD_ROLE_TOWER (10) /* Tower tile (tower file) */ -#define FD_KEYGUARD_ROLE_CNT (11) /* number of known roles */ +#define FD_KEYGUARD_ROLE_FAILOV (11) /* Failover tile member certificate and TLS */ +#define FD_KEYGUARD_ROLE_CNT (12) /* number of known roles */ + +/* The failov tile gets its member certificate signed, a plain Ed25519 + signature by the staked key over a 16 byte domain prefix followed by + the sign tile's own junk pubkey. The prefix keeps that signature out + of every other raw message domain, in particular the 32 byte shred + roots the leader role signs. The junk key signs its TLS + CertificateVerify. */ +#define FD_KEYGUARD_MEMBER_CERT_PREFIX "FD_FAILOVER_MBR1" +#define FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ (16UL) +#define FD_KEYGUARD_MEMBER_CERT_MSG_SZ (48UL) /* Payload types ******************************************************/ @@ -54,6 +65,7 @@ FD_STATIC_ASSERT( FD_KEYGUARD_AUTH_VOTERS_MAX<=16UL, auth_voters_max_fits_sign_r #define FD_KEYGUARD_PAYLOAD_LG_BLS_PUBKEY (12) /* Alpenglow BLS public key query */ #define FD_KEYGUARD_PAYLOAD_LG_TOWER (13) /* Tower file */ #define FD_KEYGUARD_PAYLOAD_LG_VOTE_HISTORY (14) /* Alpenglow vote history file */ +#define FD_KEYGUARD_PAYLOAD_LG_FAILOV (15) /* Failover member certificate */ #define FD_KEYGUARD_PAYLOAD_TXN (1UL<bls_keys[ 0 ], ctx->public_key, ctx->private_key, ctx->sha512 ); } +/* A failover switch passes only a public key, we copy the matching + key loaded at boot into the identity key. */ + +static void FD_FN_SENSITIVE +failover_select( fd_sign_ctx_t * ctx ) { + uchar const * selected = NULL; + if( FD_LIKELY( ctx->failover_staked_key ) ) { + if( fd_memeq( ctx->keyswitch->bytes, ctx->failover_junk_key+32UL, 32UL ) ) selected = ctx->failover_junk_key; + else if( fd_memeq( ctx->keyswitch->bytes, ctx->failover_staked_key+32UL, 32UL ) ) selected = ctx->failover_staked_key; + } + if( FD_UNLIKELY( !selected ) ) FD_LOG_ERR(( "identity switch selects a key that was not loaded at boot" )); + fd_memzero_explicit( ctx->keyswitch->bytes, 64UL ); + fd_memcpy( ctx->private_key, selected, 32UL ); + fd_memcpy( ctx->public_key, selected+32UL, 32UL ); + derive_fields( ctx ); + fd_keyswitch_state( ctx->keyswitch, FD_KEYSWITCH_STATE_COMPLETED ); +} + +/* failover_housekeeping runs the keyswitches the failover way before + the normal keyswitch code sees them. A failover switch selects a key + loaded at boot. */ + +static void FD_FN_SENSITIVE +failover_housekeeping( fd_sign_ctx_t * ctx ) { + if( FD_UNLIKELY( fd_keyswitch_state_query( ctx->keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING && + fd_keyswitch_param_query( ctx->keyswitch )==FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY ) ) { + failover_select( ctx ); + } + if( FD_UNLIKELY( ctx->av_keyswitch && fd_keyswitch_state_query( ctx->av_keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING && + fd_keyswitch_param_query( ctx->av_keyswitch )==FD_KEYSWITCH_PARAM_AV_ADD && + fd_memeq( ctx->av_keyswitch->bytes+32UL, ctx->failover_staked_key+32UL, 32UL ) ) ) { + FD_LOG_WARNING(( "keyswitch failed: the staked [paths.identity_key] key cannot be an authorized voter under failover" )); + fd_memzero_explicit( ctx->av_keyswitch->bytes, 64UL ); + ctx->av_keyswitch->result = FD_ADMINCTL_RESULT_UNSUPPORTED; + fd_keyswitch_state( ctx->av_keyswitch, FD_KEYSWITCH_STATE_FAILED ); + } +} + static void FD_FN_SENSITIVE during_housekeeping_sensitive( fd_sign_ctx_t * ctx ) { + if( FD_UNLIKELY( ctx->failover_staked_key ) ) failover_housekeeping( ctx ); + if( FD_UNLIKELY( fd_keyswitch_state_query( ctx->keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) { memcpy( ctx->private_key, ctx->keyswitch->bytes, 32UL ); fd_memzero_explicit( ctx->keyswitch->bytes, 32UL ); @@ -208,6 +251,25 @@ during_frag( void * _ctx, during_frag_sensitive( _ctx, in_idx, seq, sig, chunk, sz ); } +/* The failover tile's ED25519 requests. The junk key signs the + failover TLS whichever key is installed. The staked key signs the + member certificate whichever key is installed, but only over our own + junk pubkey. */ + +static void FD_FN_SENSITIVE +failover_sign( fd_sign_ctx_t * ctx, + uchar * dst, + ulong sz ) { + if( sz!=FD_KEYGUARD_MEMBER_CERT_MSG_SZ ) { + fd_ed25519_sign( dst, ctx->_data, sz, ctx->failover_junk_key+32UL, ctx->failover_junk_key, ctx->sha512 ); + return; + } + if( FD_UNLIKELY( !fd_memeq( ctx->_data+FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ, ctx->failover_junk_key+32UL, 32UL ) ) ) { + FD_LOG_EMERG(( "member certificate request is not for our junk key" )); + } + fd_ed25519_sign( dst, ctx->_data, sz, ctx->failover_staked_key+32UL, ctx->failover_staked_key, ctx->sha512 ); +} + static void FD_FN_SENSITIVE after_frag_sensitive( void * _ctx, ulong in_idx, @@ -261,6 +323,7 @@ after_frag_sensitive( void * _ctx, switch( sign_type ) { case FD_KEYGUARD_SIGN_TYPE_ED25519: { + if( FD_UNLIKELY( role==FD_KEYGUARD_ROLE_FAILOV ) ) { failover_sign( ctx, dst, sz ); break; } fd_ed25519_sign( dst, ctx->_data, sz, ctx->public_key, ctx->private_key, ctx->sha512 ); if( needs_second_sign ) { ulong authority_idx = (sig >> 33) & 0xFUL; @@ -337,13 +400,42 @@ after_frag( void * _ctx, after_frag_sensitive( _ctx, in_idx, seq, sig, sz, tsorig, tspub, stem ); } +/* failover_load_keys sets the failover keys, all NULL unless failover + is enabled. Under failover we also keep read only copies of the junk + and staked keypairs, loaded here before the sandbox. We boot with the + junk key as the identity key. */ + static void FD_FN_SENSITIVE -privileged_init_sensitive( fd_topo_t const * topo, - fd_topo_tile_t const * tile ) { - void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id ); - FD_SCRATCH_ALLOC_INIT( l, scratch ); - fd_sign_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_sign_ctx_t ), sizeof( fd_sign_ctx_t ) ); +failover_load_keys( fd_sign_ctx_t * ctx, + fd_topo_tile_t const * tile ) { + ctx->failover_junk_key = NULL; + ctx->failover_staked_key = NULL; + if( FD_LIKELY( !tile->sign.failover_enabled ) ) return; + + uchar * junk_key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); + fd_memcpy( junk_key, ctx->private_key, 64UL ); + ctx->failover_junk_key = fd_keyload_mprotect_ro( junk_key, /* public_key_only: */ 0 ); + ctx->failover_staked_key = fd_keyload_load( tile->sign.failover_staked_key_path, /* pubkey only: */ 0 ); + if( FD_UNLIKELY( fd_memeq( ctx->failover_junk_key+32UL, ctx->failover_staked_key+32UL, 32UL ) ) ) { + FD_LOG_ERR(( "the failover junk identity must differ from the [paths.identity_key] key, which is the staked identity under failover" )); + } + fd_sha512_t sha[ 1 ]; + FD_TEST( fd_sha512_join( fd_sha512_new( sha ) ) ); + uchar public_key[ 32 ]; + fd_ed25519_public_from_private( public_key, ctx->failover_staked_key, sha ); + if( FD_UNLIKELY( !fd_memeq( public_key, ctx->failover_staked_key+32UL, 32UL ) ) ) { + FD_LOG_ERR(( "the public key in [paths.identity_key] does not match its private key" )); + } + for( ulong i=0UL; iauthorized_voters_cnt; i++ ) { + if( FD_UNLIKELY( fd_memeq( ctx->authorized_voter_pubkeys[ i ], ctx->failover_staked_key+32UL, 32UL ) ) ) { + FD_LOG_ERR(( "the staked [paths.identity_key] key cannot be one of the [paths.authorized_voter_paths] under failover" )); + } + } +} +static void FD_FN_SENSITIVE +load_keys( fd_sign_ctx_t * ctx, + fd_topo_tile_t const * tile ) { uchar * identity_key = fd_keyload_mprotect_wr( fd_keyload_load( tile->sign.identity_key_path, /* pubkey only: */ 0 ), /* public_key_only: */ 0 ); ctx->private_key = identity_key; ctx->public_key = identity_key + 32UL; @@ -358,6 +450,18 @@ privileged_init_sensitive( fd_topo_t const * topo, memcpy( ctx->authorized_voter_pubkeys[ i ], authorized_voter_key + 32UL, 32UL ); } + failover_load_keys( ctx, tile ); +} + +static void FD_FN_SENSITIVE +privileged_init_sensitive( fd_topo_t const * topo, + fd_topo_tile_t const * tile ) { + void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id ); + FD_SCRATCH_ALLOC_INIT( l, scratch ); + fd_sign_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof( fd_sign_ctx_t ), sizeof( fd_sign_ctx_t ) ); + + load_keys( ctx, tile ); + /* The stack can be taken over and reorganized by under AddressSanitizer, which causes this code to fail. */ #if FD_HAS_ASAN @@ -472,6 +576,12 @@ unprivileged_init_sensitive( fd_topo_t const * topo, FD_TEST( !strcmp( out_link->name, "sign_votor" ) ); FD_TEST( in_link->mtu==FD_KEYGUARD_SIGN_REQ_MTU ); FD_TEST( out_link->mtu==FD_BLS_SIG_SZ ); + } else if( !strcmp(in_link->name, "failov_sign" ) ) { + ctx->in[ i ].role = FD_KEYGUARD_ROLE_FAILOV; + FD_TEST( !strcmp( out_link->name, "sign_failov" ) ); + FD_TEST( in_link->mtu==130UL ); + FD_TEST( out_link->mtu==64UL ); + FD_TEST( ctx->failover_staked_key ); } else { FD_LOG_CRIT(( "unexpected link %s", in_link->name )); } diff --git a/src/disco/keyguard/fuzz_keyguard.c b/src/disco/keyguard/fuzz_keyguard.c index f2ca7f0869c..18eace0288b 100644 --- a/src/disco/keyguard/fuzz_keyguard.c +++ b/src/disco/keyguard/fuzz_keyguard.c @@ -1,4 +1,5 @@ #include "fd_keyguard.h" +#include "../../waltz/tls/fd_tls.h" #include @@ -29,5 +30,17 @@ LLVMFuzzerTestOneInput( uchar const * data, (void)fd_keyguard_payload_authorize( &authority, data, size, role, (int)i ); } } + + /* The sign tile picks the failover key by size, so the failov role + signs only the member certificate or a CertificateVerify. */ + for( ulong i=0UL; i #include @@ -295,6 +296,186 @@ test_bls_request_rejected( void ) { FD_TEST( bls_request_exit_status( FD_KEYGUARD_SIGN_TYPE_BLS|(1UL<<32), 11UL )==1 ); /* authorized voter 0, none loaded */ } +/* Failover: the sign tile holds the junk and staked keypairs it loaded + at boot and switches between them by public key. */ + +static uchar junk_key [ 64 ]; +static uchar staked_key[ 64 ]; + +static void +failover_setup( void ) { + setup(); + fd_memset( junk_key, 0x33, 32UL ); + fd_memset( staked_key, 0x44, 32UL ); + fd_ed25519_public_from_private( junk_key +32UL, junk_key, ctx.sha512 ); + fd_ed25519_public_from_private( staked_key+32UL, staked_key, ctx.sha512 ); + fd_memcpy( identity_key, junk_key, 64UL ); /* a failover member boots under the junk key */ + derive_fields( &ctx ); + ctx.failover_junk_key = junk_key; + ctx.failover_staked_key = staked_key; + ctx.in[0].role = FD_KEYGUARD_ROLE_FAILOV; + client.response_mtu = FD_ED25519_SIG_SZ; +} + +static void * +sign_one( void * arg ) { + ulong seq = *(ulong *)arg; + fd_frag_meta_t const * line = client.request+fd_mcache_line_idx( seq, TEST_DEPTH ); + while( fd_frag_meta_seq_query( line )!=seq ) FD_SPIN_PAUSE(); + FD_COMPILER_MFENCE(); + during_frag( &ctx, 0UL, seq, line->sig, line->chunk, line->sz, 0UL ); + after_frag( &ctx, 0UL, seq, line->sig, line->sz, 0UL, 0UL, &stem ); + return NULL; +} + +/* Whether the sign tile signs payload with the key of public_key. */ + +static int +failover_signed_by( uchar const * payload, + ulong payload_sz, + uchar const * public_key ) { + uchar signature[ FD_ED25519_SIG_SZ ]; + ulong seq = client.request_seq; + pthread_t signer; + FD_TEST( !pthread_create( &signer, NULL, sign_one, &seq ) ); + fd_keyguard_client_sign( &client, signature, payload, payload_sz, FD_KEYGUARD_SIGN_TYPE_ED25519 ); + FD_TEST( !pthread_join( signer, NULL ) ); + return FD_ED25519_SUCCESS==fd_ed25519_verify( payload, payload_sz, signature, public_key, ctx.sha512 ); +} + +static void +failover_switch( fd_keyswitch_t * ks, + uchar const * public_key ) { + fd_memcpy( ks->bytes, public_key, 32UL ); + fd_memset( ks->bytes+32UL, 0, 32UL ); + ks->param = FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY; + fd_keyswitch_state( ks, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( ks )==FD_KEYSWITCH_STATE_COMPLETED ); + for( ulong i=0UL; i<64UL; i++ ) FD_TEST( !ks->bytes[ i ] ); +} + +/* Without failover an identity switch takes the keypair whatever the + param says, the failover key selection does not run. */ + +static void +test_identity_switch_without_failover( void ) { + setup(); + fd_keyswitch_t identity_switch[1]; + ctx.keyswitch = fd_keyswitch_join( fd_keyswitch_new( identity_switch, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.keyswitch ); + uchar keypair[ 64 ]; + fd_memset( keypair, 0x55, 32UL ); + fd_ed25519_public_from_private( keypair+32UL, keypair, ctx.sha512 ); + fd_memcpy( ctx.keyswitch->bytes, keypair, 64UL ); + ctx.keyswitch->param = FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY; + fd_keyswitch_state( ctx.keyswitch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( ctx.keyswitch )==FD_KEYSWITCH_STATE_COMPLETED ); + FD_TEST( fd_memeq( ctx.private_key, keypair, 32UL ) && fd_memeq( ctx.public_key, keypair+32UL, 32UL ) ); + + /* Adding an authorized voter takes the normal path, whatever its + key. */ + fd_keyswitch_t voter_switch[1]; + ctx.av_keyswitch = fd_keyswitch_join( fd_keyswitch_new( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); + FD_TEST( ctx.av_keyswitch ); + fd_memcpy( voter_switch->bytes, keypair, 64UL ); + voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_COMPLETED && ctx.authorized_voters_cnt==1UL ); +} + +static void +test_failover_keys( void ) { + failover_setup(); + fd_keyswitch_t identity_switch[1]; + ctx.keyswitch = fd_keyswitch_join( fd_keyswitch_new( identity_switch, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.keyswitch ); + + uchar cert[ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ]; + fd_memcpy( cert, FD_KEYGUARD_MEMBER_CERT_PREFIX, FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ ); + fd_memcpy( cert+FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ, junk_key+32UL, 32UL ); + uchar cv[ FD_TLS_CV_SIGN_SZ ]; + fd_memcpy( cv, fd_tls13_cli_sign_prefix, sizeof(fd_tls13_cli_sign_prefix) ); + fd_memset( cv+sizeof(fd_tls13_cli_sign_prefix), 0x5A, sizeof(cv)-sizeof(fd_tls13_cli_sign_prefix) ); + + fd_keyguard_bls_key_t junk_bls[1], staked_bls[1]; + fd_keyguard_bls_key_derive( junk_bls, junk_key +32UL, junk_key, ctx.sha512 ); + fd_keyguard_bls_key_derive( staked_bls, staked_key+32UL, staked_key, ctx.sha512 ); + + /* Junk, staked, then junk again. Whichever is installed, the staked + key signs the certificate and the junk key the handshake, and BLS + key 0 follows the installed identity. */ + for( ulong i=0UL; i<3UL; i++ ) { + uchar const * installed = i==1UL ? staked_key : junk_key; + fd_keyguard_bls_key_t const * bls = i==1UL ? staked_bls : junk_bls; + if( i ) failover_switch( identity_switch, installed+32UL ); + FD_TEST( fd_memeq( ctx.public_key, installed+32UL, 32UL ) ); + FD_TEST( fd_memeq( ctx.private_key, installed, 32UL ) ); + FD_TEST( fd_memeq( keys[0].public_key, bls->public_key, FD_KEYGUARD_BLS_PUBKEY_SZ ) ); + FD_TEST( failover_signed_by( cert, sizeof(cert), staked_key+32UL ) ); + FD_TEST( failover_signed_by( cv, sizeof(cv), junk_key +32UL ) ); + } + + /* The staked key is the failover identity here, so it cannot become an authorized voter. */ + fd_keyswitch_t voter_switch[1]; + ctx.av_keyswitch = fd_keyswitch_join( fd_keyswitch_new( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); + FD_TEST( ctx.av_keyswitch ); + fd_memcpy( voter_switch->bytes, staked_key, 64UL ); + voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_FAILED ); + FD_TEST( voter_switch->result==FD_ADMINCTL_RESULT_UNSUPPORTED ); + FD_TEST( !ctx.authorized_voters_cnt ); + for( ulong i=0UL; i<64UL; i++ ) FD_TEST( !voter_switch->bytes[ i ] ); + FD_LOG_NOTICE(( "pass: failover keys" )); +} + +static fd_keyswitch_t failover_ks[1]; + +static void +switch_to_unknown_key( void ) { + uchar other[ 64 ]; + fd_memset( other, 0x55, 32UL ); + fd_ed25519_public_from_private( other+32UL, other, ctx.sha512 ); + failover_switch( failover_ks, other+32UL ); +} + +static void +sign_foreign_cert( void ) { + fd_memcpy( ctx._data, FD_KEYGUARD_MEMBER_CERT_PREFIX, FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ ); + fd_memcpy( ctx._data+FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ, staked_key+32UL, 32UL ); + after_frag( &ctx, 0UL, 0UL, FD_KEYGUARD_SIGN_TYPE_ED25519, FD_KEYGUARD_MEMBER_CERT_MSG_SZ, 0UL, 0UL, &stem ); +} + +static int +child_exit_status( void (*fn)( void ) ) { + pid_t pid = fork(); + FD_TEST( pid>=0 ); + if( !pid ) { + fd_log_level_core_set( 8 ); + fn(); + _exit( 0 ); + } + int status; + FD_TEST( waitpid( pid, &status, 0 )==pid ); + FD_TEST( WIFEXITED( status ) ); + return WEXITSTATUS( status ); +} + +/* A switch to a key not loaded at boot, and a certificate over another + junk key, stop the validator. */ + +static void +test_failover_refusals( void ) { + failover_setup(); + ctx.keyswitch = fd_keyswitch_join( fd_keyswitch_new( failover_ks, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.keyswitch ); + FD_TEST( child_exit_status( switch_to_unknown_key )==1 ); + FD_TEST( child_exit_status( sign_foreign_cert )==1 ); + FD_LOG_NOTICE(( "pass: failover refusals" )); +} + int main( int argc, char ** argv ) { @@ -309,6 +490,9 @@ main( int argc, test_bls_pubkey_last_authority(); test_bls_pubkey_request_rejected(); test_bls_request_rejected(); + test_identity_switch_without_failover(); + test_failover_keys(); + test_failover_refusals(); FD_LOG_NOTICE(( "pass" )); fd_halt(); return 0; diff --git a/src/disco/topo/fd_topo.c b/src/disco/topo/fd_topo.c index a4f74c778f2..7ae13915287 100644 --- a/src/disco/topo/fd_topo.c +++ b/src/disco/topo/fd_topo.c @@ -222,6 +222,11 @@ fd_topo_tile_extra_huge_pages( fd_topo_tile_t const * tile ) { FD_FN_PURE static ulong fd_topo_tile_extra_normal_pages( fd_topo_tile_t const * tile ) { ulong key_pages = 0UL; + if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) && tile->sign.failover_enabled ) ) { + /* The read only junk and staked key pages and their guards, next + to the identity key. */ + key_pages += 10UL; + } if( FD_UNLIKELY( tile->id_keyswitch_obj_id!=ULONG_MAX ) ) { /* Certain tiles using fd_keyload_load need normal pages to hold key material. */ diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index 889b8df1021..6fb6cd1a318 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -403,6 +403,8 @@ struct fd_topo_tile { struct { char identity_key_path[ PATH_MAX ]; + int failover_enabled; + char failover_staked_key_path[ PATH_MAX ]; ulong authorized_voter_paths_cnt; char authorized_voter_paths[ FD_KEYGUARD_AUTH_VOTERS_MAX ][ PATH_MAX ]; struct { diff --git a/src/discof/admin/fd_admin_tile.c b/src/discof/admin/fd_admin_tile.c index 6fefacc7a53..a4af9f8940f 100644 --- a/src/discof/admin/fd_admin_tile.c +++ b/src/discof/admin/fd_admin_tile.c @@ -24,6 +24,7 @@ struct fd_admin_tile_ctx { long poll_ticks; long idle_poll_ticks; long deadline_ticks; /* next adminctl poll */ + int failover_enabled; /* failover moves the identity, set-identity is refused */ }; typedef struct fd_admin_tile_ctx fd_admin_tile_ctx_t; @@ -100,6 +101,7 @@ unprivileged_init( fd_topo_t const * topo, ctx->poll_ticks = (long)(1e6*fd_tempo_tick_per_ns( NULL )); /* 1ms */ ctx->idle_poll_ticks = (long)((double)FD_SLEEP_PARK_CAP_NS*fd_tempo_tick_per_ns( NULL )); ctx->deadline_ticks = 0L; + ctx->failover_enabled = fd_topo_find_tile( topo, "failov", 0UL )!=ULONG_MAX; ctx->topo = topo; fd_topo_obj_t const * adminctl_obj = fd_topo_find_tile_obj( topo, tile, "adminctl" ); @@ -577,6 +579,14 @@ set_identity( fd_admin_tile_ctx_t * ctx, FD_BASE58_ENCODE_32_BYTES( ctx->identity_pubkey, old_identity ); FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, "{\"old_identity\":\"%s\"}", old_identity ) ); + /* Under failover the failover tile moves the identity. */ + if( FD_UNLIKELY( ctx->failover_enabled ) ) { + FD_LOG_WARNING(( "set-identity is not supported while [failover.enabled] is true" )); + report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNSUPPORTED ); + fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNSUPPORTED ); + return; + } + if( FD_UNLIKELY( data_szjunk_pubkey ); + if( FD_UNLIKELY( fd_ed25519_verify( msg, sizeof(msg), hello->member_cert, hello->staked_pubkey, sha )!=FD_ED25519_SUCCESS ) ) { + return FD_FAILOVER_HELLO_ERR_CERT; + } + return FD_FAILOVER_HELLO_OK; +} + ulong fd_failover_session_init( int dial_peer ) { return dial_peer ? FD_FAILOVER_SESSION_BACKOFF : FD_FAILOVER_SESSION_LISTENING; diff --git a/src/discof/failover/fd_failover_proto.h b/src/discof/failover/fd_failover_proto.h index 8479dc023f1..9bfd06c27c2 100644 --- a/src/discof/failover/fd_failover_proto.h +++ b/src/discof/failover/fd_failover_proto.h @@ -91,6 +91,20 @@ int fd_failover_hello_check( fd_failover_hello_t const * self, fd_failover_hello_t const * peer ); +/* fd_failover_member_cert_msg writes the 48 byte message a member + certificate signs, the keyguard's member cert prefix then + junk_pubkey. The sign tile signs exactly this with the staked key. */ +void +fd_failover_member_cert_msg( uchar out[ 48 ], + uchar const junk_pubkey[ 32 ] ); + +/* fd_failover_member_cert_check returns FD_FAILOVER_HELLO_OK if the + member_cert in hello is the staked pubkey's signature over the junk + pubkey in hello, else FD_FAILOVER_HELLO_ERR_CERT. */ +int +fd_failover_member_cert_check( fd_failover_hello_t const * hello, + fd_sha512_t * sha ); + /* fd_failover_session_init returns the resting state of an endpoint, LISTENING for the listener and BACKOFF for the dialer. */ ulong From d60581399d27d03736833f2c95d1d01e008d654b Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Fri, 18 Sep 2026 17:16:46 +0000 Subject: [PATCH 06/23] failover: add the handoff messages and validate the final Tower --- src/discof/failover/fd_failover_proto.c | 112 ++++++++++++++++++++++ src/discof/failover/fd_failover_proto.h | 111 +++++++++++++++++++++ src/discof/failover/test_failover_proto.c | 19 ++++ 3 files changed, 242 insertions(+) diff --git a/src/discof/failover/fd_failover_proto.c b/src/discof/failover/fd_failover_proto.c index bcbb6be5e5a..836752325c4 100644 --- a/src/discof/failover/fd_failover_proto.c +++ b/src/discof/failover/fd_failover_proto.c @@ -90,3 +90,115 @@ fd_failover_session_step( ulong state, } return state; } + +int +fd_failover_handoff_request_decode( fd_failover_handoff_request_t * out, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( payload_sz!=sizeof(fd_failover_handoff_request_t) ) ) return 0; + fd_failover_handoff_request_t request; + fd_memcpy( &request, payload, sizeof(request) ); + if( FD_UNLIKELY( !request.handoff_id || !request.target_boot_id ) ) return 0; + *out = request; + return 1; +} + +int +fd_failover_handoff_result_decode( fd_failover_handoff_result_t * out, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( payload_sz!=sizeof(fd_failover_handoff_result_t) ) ) return 0; + fd_failover_handoff_result_t result; + fd_memcpy( &result, payload, sizeof(result) ); + if( FD_UNLIKELY( !result.handoff_id ) ) return 0; + *out = result; + return 1; +} + +ulong +fd_failover_demoted_encode( uchar * out, + ulong handoff_id, + ulong target_boot_id, + ulong last_vote_slot, + uchar const * state, + ulong state_sz ) { + if( FD_UNLIKELY( !state_sz || state_sz>FD_FAILOVER_TOWER_STATE_MAX ) ) return 0UL; + + fd_failover_demoted_t demoted = { + .handoff_id = handoff_id, + .target_boot_id = target_boot_id, + .last_vote_slot = last_vote_slot, + .mode = (uchar)FD_FAILOVER_MODE_TOWER, + .state_len = (ushort)state_sz, + }; + fd_memcpy( out, &demoted, sizeof(demoted) ); + fd_memcpy( out+sizeof(demoted), state, state_sz ); + return sizeof(demoted)+state_sz; +} + +int +fd_failover_demoted_decode( fd_failover_demoted_t * out, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( payload_szFD_FAILOVER_DEMOTED_PAYLOAD_MAX ) ) return 0; + + fd_failover_demoted_t demoted; + fd_memcpy( &demoted, payload, sizeof(demoted) ); + + ulong state_sz = payload_sz-sizeof(fd_failover_demoted_t); + if( FD_UNLIKELY( (ulong)demoted.state_len!=state_sz || + !state_sz || + demoted.mode!=(uchar)FD_FAILOVER_MODE_TOWER || + demoted.last_vote_slot==FD_FAILOVER_SLOT_NULL ) ) return 0; + + fd_compact_tower_sync_serde_t serde; + fd_tower_vote_t votes[ FD_TOWER_VOTE_MAX ]; + ulong vote_cnt; + ulong root; + if( FD_UNLIKELY( fd_compact_tower_sync_de_exact( &serde, payload+sizeof(fd_failover_demoted_t), state_sz ) || + fd_compact_tower_sync_to_votes( &serde, votes, &vote_cnt, &root ) || + !vote_cnt || votes[ vote_cnt-1UL ].slot!=demoted.last_vote_slot ) ) return 0; + + *out = demoted; + return 1; +} + +ulong +fd_failover_promote_ack_encode( uchar * out, + ulong handoff_id ) { + fd_failover_promote_ack_t ack = { .handoff_id=handoff_id }; + fd_memcpy( out, &ack, sizeof(ack) ); + return sizeof(ack); +} + +ulong +fd_failover_promote_rejected_encode( uchar * out, + ulong handoff_id, + uchar reason ) { + if( FD_UNLIKELY( reason==(uchar)FD_FAILOVER_REJECT_NONE || reason>=(uchar)FD_FAILOVER_REJECT_CNT ) ) return 0UL; + fd_failover_promote_rejected_t rej = { .handoff_id=handoff_id, .reason=reason }; + fd_memcpy( out, &rej, sizeof(rej) ); + return sizeof(rej); +} + +int +fd_failover_promote_ack_decode( fd_failover_promote_ack_t * out, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( payload_sz!=sizeof(fd_failover_promote_ack_t) ) ) return 0; + fd_memcpy( out, payload, sizeof(fd_failover_promote_ack_t) ); + return 1; +} + +int +fd_failover_promote_rejected_decode( fd_failover_promote_rejected_t * out, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( payload_sz!=sizeof(fd_failover_promote_rejected_t) ) ) return 0; + fd_failover_promote_rejected_t rej; + fd_memcpy( &rej, payload, sizeof(rej) ); + if( FD_UNLIKELY( rej.reason==(uchar)FD_FAILOVER_REJECT_NONE || rej.reason>=(uchar)FD_FAILOVER_REJECT_CNT ) ) return 0; + *out = rej; + return 1; +} diff --git a/src/discof/failover/fd_failover_proto.h b/src/discof/failover/fd_failover_proto.h index 9bfd06c27c2..5a606778dfb 100644 --- a/src/discof/failover/fd_failover_proto.h +++ b/src/discof/failover/fd_failover_proto.h @@ -82,6 +82,60 @@ struct __attribute__((packed)) fd_failover_hello { typedef struct fd_failover_hello fd_failover_hello_t; FD_STATIC_ASSERT( sizeof(fd_failover_hello_t)==192UL, wire_layout ); +/* Sent by the demoter once it runs the junk key, the final tower + follows it. */ +struct __attribute__((packed)) fd_failover_demoted { + ulong handoff_id; /* the handoff id from the standby's HANDOFF_REQUEST */ + ulong target_boot_id; /* peer boot_id the handoff is for */ + ulong last_vote_slot; /* tip of the final tower */ + uchar mode; /* FD_FAILOVER_MODE_* encoding */ + ushort state_len; /* size of the final tower that follows this struct */ +}; +typedef struct fd_failover_demoted fd_failover_demoted_t; +FD_STATIC_ASSERT( sizeof(fd_failover_demoted_t)==27UL, wire_layout ); + +#define FD_FAILOVER_DEMOTED_PAYLOAD_MAX (sizeof(fd_failover_demoted_t)+FD_FAILOVER_TOWER_STATE_MAX) + +struct __attribute__((packed)) fd_failover_promote_ack { + ulong handoff_id; +}; +typedef struct fd_failover_promote_ack fd_failover_promote_ack_t; +FD_STATIC_ASSERT( sizeof(fd_failover_promote_ack_t)==8UL, wire_layout ); + +struct __attribute__((packed)) fd_failover_promote_rejected { + ulong handoff_id; + uchar reason; /* FD_FAILOVER_REJECT_* */ +}; +typedef struct fd_failover_promote_rejected fd_failover_promote_rejected_t; +FD_STATIC_ASSERT( sizeof(fd_failover_promote_rejected_t)==9UL, wire_layout ); + +/* The standby asks the active it authenticated to hand over. */ +struct __attribute__((packed)) fd_failover_handoff_request { + ulong handoff_id; + ulong target_boot_id; + ulong replay_slot; /* standby's replay slot, FD_FAILOVER_SLOT_NULL before replay starts */ +}; +typedef struct fd_failover_handoff_request fd_failover_handoff_request_t; +FD_STATIC_ASSERT( sizeof(fd_failover_handoff_request_t)==24UL, wire_layout ); + +/* Ends the exchange, result is an admin control result. */ +struct __attribute__((packed)) fd_failover_handoff_result { + ulong handoff_id; + ulong result; +}; +typedef struct fd_failover_handoff_result fd_failover_handoff_result_t; +FD_STATIC_ASSERT( sizeof(fd_failover_handoff_result_t)==16UL, wire_layout ); + +/* PROMOTE_REJECTED reasons */ +#define FD_FAILOVER_REJECT_NONE (0U) +#define FD_FAILOVER_REJECT_BUSY (1U) +#define FD_FAILOVER_REJECT_HOLDS_IDENTITY (2U) +#define FD_FAILOVER_REJECT_REPLAY_BEHIND (3U) +#define FD_FAILOVER_REJECT_ADOPTION_MISMATCH (4U) +#define FD_FAILOVER_REJECT_ADOPTION_FAILED (5U) +#define FD_FAILOVER_REJECT_SWITCH_FAILED (6U) +#define FD_FAILOVER_REJECT_CNT (7U) + FD_PROTOTYPES_BEGIN /* Checks a peer's HELLO against ours and returns FD_FAILOVER_HELLO_OK or @@ -123,6 +177,63 @@ fd_failover_session_step( ulong state, int dial_peer, int event ); +/* Validate a handoff request or result. Both have exact wire sizes. + Handoff ids and target boot ids must be nonzero. The controller binds + replies to an outstanding request and treats an unknown result as a + refusal. Return 0 on failure without changing out. */ +int +fd_failover_handoff_request_decode( fd_failover_handoff_request_t * out, + uchar const * payload, + ulong payload_sz ); + +int +fd_failover_handoff_result_decode( fd_failover_handoff_result_t * out, + uchar const * payload, + ulong payload_sz ); + +/* Writes a DEMOTED payload, the header and then the tower, into out, + which holds FD_FAILOVER_DEMOTED_PAYLOAD_MAX bytes. Returns the payload + size, 0 for an empty or oversized tower. */ +ulong +fd_failover_demoted_encode( uchar * out, + ulong handoff_id, + ulong target_boot_id, + ulong last_vote_slot, + uchar const * state, + ulong state_sz ); + +/* Validates a DEMOTED payload. The tower has to decode exactly and end + at last_vote_slot. Returns 1 on success, the tower is then at + payload+sizeof(fd_failover_demoted_t). Returns 0 on failure with out + left unchanged. */ +int +fd_failover_demoted_decode( fd_failover_demoted_t * out, + uchar const * payload, + ulong payload_sz ); + +/* Write a PROMOTE_ACK or PROMOTE_REJECTED payload into out and return + its size. The reject encoder returns 0 for an unknown reason. */ +ulong +fd_failover_promote_ack_encode( uchar * out, + ulong handoff_id ); + +ulong +fd_failover_promote_rejected_encode( uchar * out, + ulong handoff_id, + uchar reason ); + +/* Validate a PROMOTE_ACK or PROMOTE_REJECTED payload. Return 1 on + success, 0 on failure with out left unchanged. */ +int +fd_failover_promote_ack_decode( fd_failover_promote_ack_t * out, + uchar const * payload, + ulong payload_sz ); + +int +fd_failover_promote_rejected_decode( fd_failover_promote_rejected_t * out, + uchar const * payload, + ulong payload_sz ); + FD_PROTOTYPES_END #endif /* HEADER_fd_src_discof_failover_fd_failover_proto_h */ diff --git a/src/discof/failover/test_failover_proto.c b/src/discof/failover/test_failover_proto.c index 42d0348214a..7ea44cf91fa 100644 --- a/src/discof/failover/test_failover_proto.c +++ b/src/discof/failover/test_failover_proto.c @@ -34,11 +34,30 @@ test_wrong_staked_key( void ) { FD_TEST( fd_failover_hello_check( &active, &standby )==FD_FAILOVER_HELLO_ERR_STAKED ); } +/* test_handoff_decode: requests and results need their exact size and + nonzero ids. */ +static void +test_handoff_decode( void ) { + fd_failover_handoff_request_t request = { .handoff_id=5UL, .target_boot_id=6UL, .replay_slot=FD_FAILOVER_SLOT_NULL }, request_out; + FD_TEST( fd_failover_handoff_request_decode( &request_out, (uchar const *)&request, sizeof(request) ) && request_out.handoff_id==5UL ); + FD_TEST( request_out.replay_slot==FD_FAILOVER_SLOT_NULL ); + FD_TEST( !fd_failover_handoff_request_decode( &request_out, (uchar const *)&request, sizeof(request)-1UL ) ); + request.target_boot_id = 0UL; + FD_TEST( !fd_failover_handoff_request_decode( &request_out, (uchar const *)&request, sizeof(request) ) ); + + fd_failover_handoff_result_t result = { .handoff_id=5UL }, result_out; + FD_TEST( fd_failover_handoff_result_decode( &result_out, (uchar const *)&result, sizeof(result) ) && result_out.handoff_id==5UL ); + FD_TEST( !fd_failover_handoff_result_decode( &result_out, (uchar const *)&result, sizeof(result)+1UL ) ); + result.handoff_id = 0UL; + FD_TEST( !fd_failover_handoff_result_decode( &result_out, (uchar const *)&result, sizeof(result) ) ); +} + int main( int argc, char ** argv ) { fd_boot( &argc, &argv ); test_hello(); test_wrong_staked_key(); + test_handoff_decode(); FD_LOG_NOTICE(( "pass" )); fd_halt(); return 0; From 2c5ba79991dadb182f1fa0b75589d02635a9b2bb Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Wed, 9 Sep 2026 19:59:33 +0000 Subject: [PATCH 07/23] failover: add the authenticated on-demand pair channel --- src/discof/failover/Local.mk | 5 + src/discof/failover/fd_failover_channel.c | 800 ++++++++++++++++++++ src/discof/failover/fd_failover_channel.h | 139 ++++ src/discof/failover/fd_failover_tls.c | 173 +++++ src/discof/failover/fd_failover_tls.h | 103 +++ src/discof/failover/test_failover_channel.c | 106 +++ 6 files changed, 1326 insertions(+) create mode 100644 src/discof/failover/fd_failover_channel.c create mode 100644 src/discof/failover/fd_failover_channel.h create mode 100644 src/discof/failover/fd_failover_tls.c create mode 100644 src/discof/failover/fd_failover_tls.h create mode 100644 src/discof/failover/test_failover_channel.c diff --git a/src/discof/failover/Local.mk b/src/discof/failover/Local.mk index 987323b173b..afacd4d010b 100644 --- a/src/discof/failover/Local.mk +++ b/src/discof/failover/Local.mk @@ -3,8 +3,13 @@ $(call add-objs,fd_failover_proto,fd_discof) $(call add-objs,fd_failover_wire,fd_discof) ifdef FD_HAS_HOSTED +$(call add-hdrs,fd_failover_channel.h) +$(call add-objs,fd_failover_channel,fd_discof) +$(call add-objs,fd_failover_tls,fd_discof) $(call make-unit-test,test_failover_proto,test_failover_proto,fd_discof fd_choreo fd_flamenco fd_tango fd_ballet fd_util) $(call run-unit-test,test_failover_proto) $(call make-unit-test,test_failover_wire,test_failover_wire,fd_discof fd_ballet fd_util) $(call run-unit-test,test_failover_wire) +$(call make-unit-test,test_failover_channel,test_failover_channel,fd_discof fd_disco fd_choreo fd_flamenco fd_waltz fd_tls fd_tango fd_ballet fd_util) +$(call run-unit-test,test_failover_channel) endif diff --git a/src/discof/failover/fd_failover_channel.c b/src/discof/failover/fd_failover_channel.c new file mode 100644 index 00000000000..547333bbe79 --- /dev/null +++ b/src/discof/failover/fd_failover_channel.c @@ -0,0 +1,800 @@ +#define _GNU_SOURCE +#include "fd_failover_channel.h" +#include "fd_failover_tls.h" +#include "../../ballet/base58/fd_base58.h" +#include "../../util/fd_util.h" +#include "../../util/net/fd_ip4.h" + +#include +#include +#include +#include +#include + +#define SOURCE_CNT (256UL) +#define PHASE_CONNECT (0) +#define PHASE_TLS (1) +#define PHASE_HELLO (2) +#define PHASE_READY (3) + +/* A HELLO whose junk key does not match the key TLS authenticated */ +#define HELLO_ERR_TLS_KEY (-1) + +struct candidate { + int fd; + int phase; + int dialed; + uint address; + long deadline; + fd_failover_tls_t tls; + fd_failover_wire_session_t wire; + fd_failover_hello_t hello; + uchar rx[ FD_FAILOVER_FRAME_MAX ]; + uchar tx[ FD_FAILOVER_FRAME_MAX ]; + ulong rx_used; + ulong tx_used; + ulong tx_sent; +}; + +struct source { + uint address; + int used; + long updated; + ulong credit; +}; + +struct fd_failover_channel { + ulong magic; /* FD_FAILOVER_CHANNEL_MAGIC once initialized */ + int dial_peer; /* the session machine runs as the dialer */ + ulong state; /* session state, stepped by fd_failover_session_step */ + uint peer_addr; /* the address a handoff dials, zero when we do not dial */ + ushort peer_port; /* the port a handoff dials */ + int listen_fd; /* listening socket, -1 until it is open */ + int paired_idx; /* the paired candidate, -1 while none is paired */ + ulong cursor; /* the next candidate to service, round robin */ + fd_failover_tls_ctx_t tls_ctx; /* TLS template every connection starts from */ + fd_failover_hello_t self_hello; /* the HELLO we send */ + fd_failover_hello_t peer_hello; /* the paired peer's HELLO, zero while unpaired */ + int member_cert_set; /* self_hello carries a checked member certificate */ + fd_sha512_t sha[ 1 ]; /* scratch for checking member certificates */ + struct candidate candidates[ FD_FAILOVER_CHANNEL_CANDIDATE_MAX ]; /* connections that are handshaking or paired */ + struct source sources[ SOURCE_CNT ]; /* accept rate limit per source address */ + long rate_updated; /* when rate_credit was last refilled */ + ulong rate_credit; /* accept rate limit across all addresses */ + long hello_timeout; /* how long a candidate has to pair */ + long silence_timeout; /* a paired session silent for this long is dropped */ + long last_rx; /* when the paired session last received a frame */ + long retry_at; /* when the dialer dials again */ + long paired_at; /* when the current session paired */ + long backoff; /* the current dial backoff, doubles up to backoff_max */ + long backoff_min; /* the shortest dial backoff, also the jitter range */ + long backoff_max; /* the longest dial backoff */ + long dial_log_at[ 3 ]; /* connect, early close, handshake timeout */ + long tls_log_at[ 4 ]; /* transport, ALPN, authentication, other */ + long hello_log_at[ 14 ]; /* one bounded bucket per HELLO refusal */ + long loss_log_at[ 6 ]; /* transport, silence, TLS, protocol, framing, local encode */ + long pair_log_at; /* when a pairing may be logged again */ + int expected_close; /* RESULT was sent, peer may close normally */ + ulong generation; /* identifies the current authenticated session */ +}; + +FD_FN_CONST ulong fd_failover_channel_align ( void ) { return alignof(fd_failover_channel_t); } +FD_FN_CONST ulong fd_failover_channel_footprint( void ) { return sizeof (fd_failover_channel_t); } + +void * +fd_failover_channel_new( void * shmem ) { + if( !shmem || !fd_ulong_is_aligned( (ulong)shmem, fd_failover_channel_align() ) ) return NULL; + fd_failover_channel_t * ch = shmem; + fd_memset( ch, 0, sizeof(*ch) ); + ch->listen_fd = -1; + ch->paired_idx = -1; + for( ulong i=0; icandidates[i].fd = -1; + ch->hello_timeout = FD_FAILOVER_CHANNEL_HELLO_TIMEOUT_NANOS; + ch->silence_timeout = FD_FAILOVER_CHANNEL_IDLE_NANOS; + ch->backoff_min = FD_FAILOVER_CHANNEL_BACKOFF_MIN_NANOS; + ch->backoff_max = FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS; + ch->backoff = ch->backoff_min; + ch->rate_credit = 16UL*1000000000UL; + FD_TEST( fd_sha512_join( fd_sha512_new( ch->sha ) ) ); + ch->magic = FD_FAILOVER_CHANNEL_MAGIC; + return shmem; +} + +fd_failover_channel_t * +fd_failover_channel_join( void * shmem ) { + if( !shmem || !fd_ulong_is_aligned( (ulong)shmem, fd_failover_channel_align() ) ) return NULL; + fd_failover_channel_t * ch = shmem; + return ch->magic==FD_FAILOVER_CHANNEL_MAGIC ? ch : NULL; +} + +static void +close_candidate( struct candidate * c ) { + fd_failover_tls_fini( &c->tls ); + if( c->fd!=-1 ) close( c->fd ); + c->fd = -1; + c->dialed = 0; + c->rx_used = c->tx_used = c->tx_sent = 0UL; + fd_memzero_explicit( &c->wire, sizeof(c->wire) ); +} + +static void +reset( fd_failover_channel_t * ch ) { + for( ulong i=0; icandidates[i] ); + ch->paired_idx = -1; + ch->last_rx = 0L; + fd_memset( &ch->peer_hello, 0, sizeof(ch->peer_hello) ); +} + +/* The controller sets a dial address only for an open handoff. */ +static int +want_dial( fd_failover_channel_t const * ch ) { + return !!ch->peer_addr; +} + +/* Resets an unpaired channel, to LISTENING without a dial address or to + BACKOFF with an immediate retry with one. */ +static void +rest( fd_failover_channel_t * ch ) { + if( ch->paired_idx>=0 ) return; + for( ulong i=0; icandidates[i].dialed ) close_candidate( &ch->candidates[i] ); + ch->dial_peer = want_dial( ch ); + ch->state = fd_failover_session_init( ch->dial_peer ); + ch->retry_at = 0L; +} + +void +fd_failover_channel_init_listener( fd_failover_channel_t * ch, + uint address, + ushort port ) { + reset( ch ); + rest( ch ); + if( ch->listen_fd!=-1 ) return; + ch->listen_fd = socket( AF_INET, SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC, 0 ); + if( ch->listen_fd==-1 ) FD_LOG_ERR(( "failover socket failed (%i-%s)", errno, fd_io_strerror( errno ) )); + int yes = 1; + if( setsockopt( ch->listen_fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes) ) ) FD_LOG_ERR(( "failover setsockopt failed" )); + struct sockaddr_in addr = { .sin_family=AF_INET, .sin_port=fd_ushort_bswap( port ), .sin_addr.s_addr=address }; + if( bind( ch->listen_fd, fd_type_pun( &addr ), sizeof(addr) ) || listen( ch->listen_fd, 64 ) ) + FD_LOG_ERR(( "could not listen on `" FD_IP4_ADDR_FMT ":%hu` from [failover.listen_address] and [failover.listen_port] (%i-%s)", + FD_IP4_ADDR_FMT_ARGS( address ), port, errno, fd_io_strerror( errno ) )); +} + +void +fd_failover_channel_init_dialer( fd_failover_channel_t * ch, + uint address, + ushort port ) { + if( address==ch->peer_addr && port==ch->peer_port ) return; + ch->peer_addr = address; + ch->peer_port = port; + if( address ) { + fd_memset( ch->dial_log_at, 0, sizeof(ch->dial_log_at) ); + /* A new dial target starts from the shortest backoff. */ + ch->backoff = ch->backoff_min; + } + rest( ch ); +} + +int +fd_failover_channel_set_identity( fd_failover_channel_t * ch, + uchar const junk_pubkey[ 32 ], + fd_tls_sign_t signer, + fd_failover_hello_t const * hello ) { + if( !fd_memeq( junk_pubkey, hello->junk_pubkey, 32UL ) || + fd_memeq( hello->junk_pubkey, hello->staked_pubkey, 32UL ) ) return -1; + reset( ch ); + ch->state = fd_failover_session_init( ch->dial_peer ); + ch->self_hello = *hello; + ch->member_cert_set = 0; + return fd_failover_tls_ctx_init( &ch->tls_ctx, junk_pubkey, signer ); +} + +int +fd_failover_channel_set_member_cert( fd_failover_channel_t * ch, + uchar const * member_cert ) { + fd_failover_hello_t hello = ch->self_hello; + fd_memcpy( hello.member_cert, member_cert, 64UL ); + if( FD_UNLIKELY( fd_failover_member_cert_check( &hello, ch->sha )!=FD_FAILOVER_HELLO_OK ) ) return -1; + ch->self_hello = hello; + ch->member_cert_set = 1; + return 0; +} + +void +fd_failover_channel_fini( fd_failover_channel_t * ch ) { + reset( ch ); + if( ch->listen_fd!=-1 ) close( ch->listen_fd ); + ch->listen_fd = -1; + fd_failover_tls_ctx_fini( &ch->tls_ctx ); +} + +void +fd_failover_channel_set_role( fd_failover_channel_t * ch, + ulong role ) { + ch->self_hello.role = (uchar)role; +} + +FD_FN_PURE ulong fd_failover_channel_state ( fd_failover_channel_t const * ch ) { return ch->state; } +FD_FN_PURE int fd_failover_channel_listen_fd( fd_failover_channel_t const * ch ) { return ch->listen_fd; } + +FD_FN_PURE int +fd_failover_channel_tx_pending( fd_failover_channel_t const * ch ) { + if( ch->paired_idx<0 ) return 0; + struct candidate const * c = &ch->candidates[ch->paired_idx]; + return !!c->tx_used || fd_tlsrec_sock_tx_pending( &c->tls.sock ); +} + +FD_FN_PURE ulong +fd_failover_channel_pending( fd_failover_channel_t const * ch ) { + ulong n = 0UL; + for( ulong i=0; icandidates[i].fd!=-1 && (int)i!=ch->paired_idx; + return n; +} + +FD_FN_PURE fd_failover_hello_t const * fd_failover_channel_peer_hello( fd_failover_channel_t const * ch ) { return &ch->peer_hello; } +FD_FN_PURE ulong fd_failover_channel_generation( fd_failover_channel_t const * ch ) { return ch->generation; } + +ushort +fd_failover_channel_listen_port( fd_failover_channel_t const * ch ) { + struct sockaddr_in addr; + socklen_t len = sizeof(addr); + if( getsockname( ch->listen_fd, fd_type_pun( &addr ), &len ) ) FD_LOG_ERR(( "failover getsockname failed" )); + return fd_ushort_bswap( addr.sin_port ); +} + +/* Spreads a redial over up to one minimum backoff, so standbys that + dial each other do not keep colliding in step. */ +static long +jitter( fd_failover_channel_t const * ch ) { + ulong r = 0UL; + if( FD_UNLIKELY( ch->backoff_min<=0L || !fd_rng_secure( &r, sizeof(r) ) ) ) return 0L; + return (long)( r%(ulong)ch->backoff_min ); +} + +/* An ALPN or authentication failure still gets logged while another + kind of TLS error is being suppressed. The number of buckets is + fixed, so unknown peers cannot allocate log state or flood it. */ +static ulong +tls_log_kind( uint reason ) { + if( !reason ) return 0UL; + if( reason==FD_TLS_REASON_ALPN_PARSE || reason==FD_TLS_REASON_ALPN_NEG || reason==FD_TLS_REASON_NO_ALPN ) return 1UL; + if( reason==FD_TLS_REASON_WRONG_PUBKEY || reason==FD_TLS_REASON_ED25519_FAIL || reason==FD_TLS_REASON_CERT_VERIFY ) return 2UL; + return 3UL; +} + +/* Every session state change goes through fd_failover_session_init or + fd_failover_session_step in fd_failover_proto.c. */ +static void +drop( fd_failover_channel_t * ch, + ulong idx, + long now, + int event ) { + struct candidate * c = &ch->candidates[idx]; + int paired = ch->paired_idx==(int)idx; + int accepted = c->fd!=-1 && !c->dialed; + close_candidate( c ); + if( paired ) { + ch->paired_idx = -1; + ch->last_rx = 0L; + fd_memset( &ch->peer_hello, 0, sizeof(ch->peer_hello) ); + } + /* A candidate from the listener does not move the session until it + pairs. */ + if( !paired && accepted ) return; + ch->state = fd_failover_session_step( ch->state, ch->dial_peer, event ); + if( paired && want_dial( ch )!=ch->dial_peer ) rest( ch ); + if( ch->dial_peer ) { + /* A session counts as established once it outlives the handshake + window, whichever side dialed it. Losing an established session + redials at once and resets the backoff, losing any other doubles + it, so a peer that rejects us right after HELLO cannot drive a + redial storm. */ + int established = paired && now>=ch->paired_at && now-ch->paired_at>=ch->hello_timeout; + ch->retry_at = fd_long_sat_add( now, fd_long_sat_add( established ? 0L : ch->backoff, jitter( ch ) ) ); + ch->backoff = established ? ch->backoff_min : fd_long_min( fd_long_sat_add( ch->backoff, ch->backoff ), ch->backoff_max ); + } +} + +enum { + LOSS_TRANSPORT, + LOSS_SILENCE, + LOSS_TLS, + LOSS_PROTOCOL, + LOSS_FRAME, + LOSS_LOCAL_ENCODE +}; + +/* Same as drop, and if idx is the paired session we log why it ended. */ +static void +lose( fd_failover_channel_t * ch, + ulong idx, + long now, + int event, + ulong cause, + char const * why ) { + if( ch->paired_idx==(int)idx && now>=ch->loss_log_at[cause] ) { + ch->loss_log_at[cause] = fd_long_sat_add( now, 60000000000L ); + FD_LOG_WARNING(( "lost the failover session with `" FD_IP4_ADDR_FMT "`, %s, repeated failures limited to one line per minute", + FD_IP4_ADDR_FMT_ARGS( ch->candidates[idx].address ), why )); + } + drop( ch, idx, now, event ); +} + +void +fd_failover_channel_hangup( fd_failover_channel_t * ch, + long now ) { + if( ch->paired_idx>=0 ) drop( ch, (ulong)ch->paired_idx, now, FD_FAILOVER_EV_LINK_LOST ); + for( ulong i=0; icandidates[i].fd!=-1 ) drop( ch, i, now, FD_FAILOVER_EV_LINK_LOST ); +} + +void +fd_failover_channel_protocol_error( fd_failover_channel_t * ch, + long now ) { + if( FD_LIKELY( ch->paired_idx>=0 ) ) { + lose( ch, (ulong)ch->paired_idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_PROTOCOL, "it sent a message that breaks the protocol, check that both machines run the same Firedancer version" ); + } + fd_failover_channel_hangup( ch, now ); +} + +static ulong +refill( ulong credit, + long * updated, + long now, + ulong rate, + ulong burst ) { + if( now>*updated ) { + /* Bound elapsed time before multiplying the scaled token count. */ + ulong elapsed = fd_ulong_min( (ulong)now-(ulong)*updated, 1000000000UL ); + credit = fd_ulong_min( credit+elapsed*rate, burst*1000000000UL ); + *updated = now; + } + return credit; +} + +static int +admit( fd_failover_channel_t * ch, + uint address, + long now ) { + ch->rate_credit = refill( ch->rate_credit, &ch->rate_updated, now, 32UL, 16UL ); + if( ch->rate_credit<1000000000UL ) return 0; + ulong same = 0UL; + for( ulong i=0; icandidates[i].fd!=-1 && ch->candidates[i].address==address; + if( same>=2UL ) return 0; + struct source * source = NULL; + struct source * reusable = NULL; + for( ulong i=0; isources[i]; + if( s->used ) s->credit = refill( s->credit, &s->updated, now, 4UL, 2UL ); + if( s->used && s->address==address ) { source = s; break; } + if( !s->used || s->credit==2000000000UL ) reusable = s; + } + if( !source ) { + if( !reusable ) return 0; + source = reusable; + *source = (struct source){ .address=address, .used=1, .updated=now, .credit=2000000000UL }; + } + if( source->credit<1000000000UL ) return 0; + source->credit -= 1000000000UL; + ch->rate_credit -= 1000000000UL; + return 1; +} + +/* A full table does not keep our dial out. The oldest candidate from + the listener makes room, so holding every slot only ever costs the + holder a slot. */ +static ulong +dial_slot( fd_failover_channel_t * ch, + long now ) { + ulong victim = FD_FAILOVER_CHANNEL_CANDIDATE_MAX; + for( ulong i=0; icandidates[i]; + if( c->fd==-1 ) return i; + if( (int)i==ch->paired_idx || c->dialed ) continue; + if( victim==FD_FAILOVER_CHANNEL_CANDIDATE_MAX || c->deadlinecandidates[victim].deadline ) victim = i; + } + if( victim==FD_FAILOVER_CHANNEL_CANDIDATE_MAX ) return victim; + drop( ch, victim, now, FD_FAILOVER_EV_TIMEOUT ); + return victim; +} + +/* A full table admits a newcomer the listener let in by closing an + inbound candidate that has not authenticated. The address holding the + most candidates loses its oldest first, so a flood of connections + mostly displaces itself, and a candidate that alone holds its address + goes only once every address holds one. */ +static ulong +listen_slot( fd_failover_channel_t * ch, + long now ) { + ulong victim = FD_FAILOVER_CHANNEL_CANDIDATE_MAX; + ulong victim_cnt = 0UL; + for( ulong i=0; icandidates[i]; + if( c->fd==-1 ) return i; + if( (int)i==ch->paired_idx || c->dialed || c->phase==PHASE_READY ) continue; + ulong cnt = 0UL; + for( ulong j=0; jcandidates[j]; + cnt += o->fd!=-1 && !o->dialed && o->address==c->address; + } + if( victim==FD_FAILOVER_CHANNEL_CANDIDATE_MAX || cnt>victim_cnt || + ( cnt==victim_cnt && c->deadlinecandidates[victim].deadline ) ) { + victim = i; + victim_cnt = cnt; + } + } + if( victim==FD_FAILOVER_CHANNEL_CANDIDATE_MAX ) return victim; + drop( ch, victim, now, FD_FAILOVER_EV_TIMEOUT ); + return victim; +} + +static int +start( fd_failover_channel_t * ch, + ulong idx, + int fd, + uint address, + long now, + int phase ) { + struct candidate * c = &ch->candidates[idx]; + c->fd = fd; c->address = address; c->phase = phase; c->dialed = phase==PHASE_CONNECT; + c->deadline = fd_long_sat_add( now, ch->hello_timeout ); + fd_failover_wire_session_init( &c->wire ); + if( fd_failover_tls_new( &c->tls, &ch->tls_ctx, fd, c->dialed ) ) { + drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); + return -1; + } + return 0; +} + +static void +accept_candidates( fd_failover_channel_t * ch, + long now, + int * busy ) { + for( ulong n=0; n<8UL; n++ ) { + struct sockaddr_in addr; + socklen_t len = sizeof(addr); + int fd = accept4( ch->listen_fd, fd_type_pun( &addr ), &len, SOCK_NONBLOCK|SOCK_CLOEXEC ); + if( fd==-1 ) return; + *busy = 1; + if( ch->paired_idx>=0 || len!=sizeof(addr) || addr.sin_family!=AF_INET || !admit( ch, addr.sin_addr.s_addr, now ) ) { + close( fd ); + continue; + } + ulong idx = listen_slot( ch, now ); + if( FD_UNLIKELY( idx==FD_FAILOVER_CHANNEL_CANDIDATE_MAX ) ) { + close( fd ); + continue; + } + if( !start( ch, idx, fd, addr.sin_addr.s_addr, now, PHASE_TLS ) ) + ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_PEER_CONNECTED ); + } +} + +static int +flush( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy ) { + struct candidate * c = &ch->candidates[idx]; + if( !c->tx_used ) { + int rc = fd_tlsrec_sock_flush( &c->tls.sock, c->fd ); + if( rc<0 ) { lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_TRANSPORT, "flushing TLS output failed" ); return -1; } + return rc; + } + long n = fd_failover_tls_write( &c->tls, c->tx+c->tx_sent, c->tx_used-c->tx_sent ); + if( n<0L ) { lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_TRANSPORT, "writing to it failed, check that the peer runs and that the machines can reach each other" ); return -1; } + if( !n ) return 1; + *busy = 1; + c->tx_sent += (ulong)n; + if( c->tx_senttx_used ) return 1; + c->tx_used = c->tx_sent = 0UL; + return fd_tlsrec_sock_tx_pending( &c->tls.sock ) ? 1 : 0; +} + +static int +decode( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy, + ushort * type, + uchar * payload, + ulong * payload_sz ) { + struct candidate * c = &ch->candidates[idx]; + uchar const * data; + ulong frame_sz; + int err = fd_failover_wire_decode( &c->wire, c->rx, c->rx_used, type, &data, payload_sz, &frame_sz ); + if( err==FD_FAILOVER_WIRE_ERR_AGAIN ) return 0; + if( err ) { lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_FRAME, "it sent a malformed frame, check that both machines run the same Firedancer version" ); return -1; } + fd_memcpy( payload, data, *payload_sz ); + memmove( c->rx, c->rx+frame_sz, c->rx_used-frame_sz ); + c->rx_used -= frame_sz; + *busy = 1; + return 1; +} + +/* We log a HELLO refusal by name, with what the operator should check. */ +static char const * +hello_err_name( int err ) { + switch( err ) { + case HELLO_ERR_TLS_KEY: return "its junk key is not the key TLS authenticated, check who can reach the failover port"; + case FD_FAILOVER_HELLO_ERR_VERSION: return "the protocol version differs, run the same Firedancer version on both machines"; + case FD_FAILOVER_HELLO_ERR_STAKED: return "the staked identity differs, check that [paths.identity_key] is the same keypair on both machines"; + case FD_FAILOVER_HELLO_ERR_VOTE_ACCT: return "the vote account differs, check that [paths.vote_account] is the same on both machines"; + case FD_FAILOVER_HELLO_ERR_JUNK_EQ: return "its junk key is ours, check that the address is the other machine, and if both machines share one [paths.base]/junk-identity.json delete it on one of them and restart that machine, never copy that file"; + case FD_FAILOVER_HELLO_ERR_JUNK_STAKE: return "a junk key is the staked key, [paths.base]/junk-identity.json must not be the staked [paths.identity_key]"; + case FD_FAILOVER_HELLO_ERR_BOTH_ACT: return "both members are active, only one machine may run the staked identity, stop signing now on the machine that should not vote with `failover demote` there"; + case FD_FAILOVER_HELLO_ERR_ROLE: return "the role is unknown, run the same Firedancer version on both machines"; + case FD_FAILOVER_HELLO_ERR_BOOT_ID: return "the boot id is zero, run the same Firedancer version on both machines"; + case FD_FAILOVER_HELLO_ERR_MODE: return "the consensus mode differs, run both machines in the same consensus mode"; + case FD_FAILOVER_HELLO_ERR_CERT: return "its member certificate is not signed by our staked key, check that [paths.identity_key] is the same keypair on both machines"; + default: return "unknown"; + } +} + +static int +read_frame( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy, + ushort * type, + uchar * payload, + ulong * payload_sz ) { + struct candidate * c = &ch->candidates[idx]; + int rc = decode( ch, idx, now, busy, type, payload, payload_sz ); + if( rc ) return rc; + ulong max = ch->paired_idx==(int)idx ? sizeof(c->rx) : FD_FAILOVER_FRAME_HDR_SZ+sizeof(fd_failover_hello_t); + if( c->rx_used>=max ) { drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); return -1; } + long n = fd_failover_tls_read( &c->tls, c->rx+c->rx_used, max-c->rx_used ); + if( n<0L ) { + /* The listener responds only to a HELLO it accepts, so a close here + usually means it refused ours. */ + if( FD_UNLIKELY( c->dialed && c->phase==PHASE_HELLO && now>=ch->dial_log_at[1] ) ) { + ch->dial_log_at[1] = fd_long_sat_add( now, 60000000000L ); + FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` closed the connection before answering our HELLO, its log says why it refused us", + FD_IP4_ADDR_FMT_ARGS( c->address ) )); + } + if( ch->paired_idx==(int)idx && ch->expected_close && c->tls.peer_closed && !fd_failover_channel_tx_pending( ch ) ) { + drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); + return -1; + } + if( fd_tlsrec_conn_is_failed( &c->tls.conn ) ) { + char why[ 192 ]; + fd_cstr_printf( why, sizeof(why), NULL, "TLS record rejected (%s), check the peer log and matching protocol versions", + fd_tls_reason_cstr( c->tls.conn.hs.base.reason ) ); + lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_TLS, why ); + } else { + lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_TRANSPORT, "the connection closed, check that the peer runs and that the machines can reach each other" ); + } + return -1; + } + if( n>0L ) { c->rx_used += (ulong)n; *busy = 1; } + return decode( ch, idx, now, busy, type, payload, payload_sz ); +} + +/* Return one to continue this poll, zero to stop after a wait or failure. */ +static int +connect_candidate( fd_failover_channel_t * ch, + ulong idx, + long now ) { + struct candidate * c = &ch->candidates[idx]; + struct sockaddr_in addr = { .sin_family=AF_INET, .sin_port=fd_ushort_bswap( ch->peer_port ), .sin_addr.s_addr=ch->peer_addr }; + if( connect( c->fd, fd_type_pun( &addr ), sizeof(addr) ) && errno!=EISCONN ) { + if( errno==EINPROGRESS || errno==EALREADY || errno==EINTR ) return 0; + if( now>=ch->dial_log_at[0] ) { + ch->dial_log_at[0] = fd_long_sat_add( now, 60000000000L ); + FD_LOG_WARNING(( "could not connect to the failover peer at `" FD_IP4_ADDR_FMT ":%hu` (%i-%s), check that it runs and that the machines can reach each other on [failover.listen_port]", + FD_IP4_ADDR_FMT_ARGS( ch->peer_addr ), ch->peer_port, errno, fd_io_strerror( errno ) )); + } + drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); + return 0; + } + c->phase = PHASE_TLS; + ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_CONNECTED ); + return 1; +} + +static int +handshake_candidate( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy ) { + struct candidate * c = &ch->candidates[idx]; + int rc = fd_failover_tls_handshake( &c->tls ); + if( rc<0 ) { + uint reason = c->tls.conn.hs.base.reason; + ulong kind = tls_log_kind( reason ); + if( now>=ch->tls_log_at[kind] ) { + ch->tls_log_at[kind] = fd_long_sat_add( now, 60000000000L ); + FD_LOG_WARNING(( "TLS handshake with `" FD_IP4_ADDR_FMT "` failed (%s), not necessarily the other failover machine, check peer reachability and junk keys on both machines", + FD_IP4_ADDR_FMT_ARGS( c->address ), reason ? fd_tls_reason_cstr( reason ) : "connection closed or socket error" )); + } + drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); + return 0; + } + if( !rc ) return 0; + *busy = 1; + c->phase = PHASE_HELLO; + /* The dialer sends first. On the listener we respond only to a HELLO + that passed the checks below. */ + if( c->dialed ) c->tx_used = fd_failover_wire_encode( &c->wire, c->tx, FD_FAILOVER_MSG_HELLO, + (uchar const *)&ch->self_hello, sizeof(ch->self_hello) ); + return 1; +} + +static int +read_candidate_hello( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy, + uchar * payload ) { + struct candidate * c = &ch->candidates[idx]; + ushort type; + ulong sz; + if( read_frame( ch, idx, now, busy, &type, payload, &sz )<=0 ) return 0; + if( type!=FD_FAILOVER_MSG_HELLO || sz!=sizeof(c->hello) ) { + if( now>=ch->hello_log_at[0] ) { + ch->hello_log_at[0] = fd_long_sat_add( now, 60000000000L ); + FD_LOG_WARNING(( "rejected the failover HELLO from `" FD_IP4_ADDR_FMT "`, its first frame is not a HELLO, check who can reach the failover port", FD_IP4_ADDR_FMT_ARGS( c->address ) )); + } + drop( ch, idx, now, FD_FAILOVER_EV_HELLO_FATAL ); + return 0; + } + fd_memcpy( &c->hello, payload, sizeof(c->hello) ); + /* The junk key in HELLO has to be the key TLS authenticated, and the + member certificate proves the peer holds the staked key. */ + int err = fd_memeq( c->hello.junk_pubkey, c->tls.peer_pubkey, 32UL ) ? fd_failover_hello_check( &ch->self_hello, &c->hello ) : HELLO_ERR_TLS_KEY; + if( err==FD_FAILOVER_HELLO_OK ) err = fd_failover_member_cert_check( &c->hello, ch->sha ); + if( err!=FD_FAILOVER_HELLO_OK ) { + ulong bucket = err==HELLO_ERR_TLS_KEY ? 12UL : ( err>0 && err<=FD_FAILOVER_HELLO_ERR_CERT ? (ulong)err : 13UL ); + if( now>=ch->hello_log_at[bucket] ) { + ch->hello_log_at[bucket] = fd_long_sat_add( now, 60000000000L ); + FD_LOG_WARNING(( "rejected the failover HELLO from `" FD_IP4_ADDR_FMT "`, %s", FD_IP4_ADDR_FMT_ARGS( c->address ), hello_err_name( err ) )); + } + drop( ch, idx, now, FD_FAILOVER_EV_HELLO_FATAL ); + return 0; + } + c->phase = PHASE_READY; + if( !c->dialed ) { + c->tx_used = fd_failover_wire_encode( &c->wire, c->tx, FD_FAILOVER_MSG_HELLO, + (uchar const *)&ch->self_hello, sizeof(ch->self_hello) ); + if( flush( ch, idx, now, busy )!=0 ) return 0; + } + return 1; +} + +static void +pair_candidate( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy ) { + struct candidate * c = &ch->candidates[idx]; + /* The first session that authenticates wins. One that came in on the + listener while we were dialing pairs as the listener. */ + for( ulong i=0; icandidates[i] ); + if( !c->dialed && ch->dial_peer ) { + ch->dial_peer = 0; + ch->state = fd_failover_session_init( ch->dial_peer ); + } + ch->paired_idx = (int)idx; + c->tls.paired = 1; + ch->peer_hello = c->hello; + ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_HELLO_OK ); + ch->generation++; + ch->last_rx = now; + ch->paired_at = now; + ch->expected_close = 0; + *busy = 1; + if( now>=ch->pair_log_at ) { + ch->pair_log_at = fd_long_sat_add( now, 60000000000L ); + char member[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( c->hello.junk_pubkey, NULL, member ); + FD_LOG_NOTICE(( "paired with the failover peer at `" FD_IP4_ADDR_FMT "` via %s, authenticated member %s, role %s, boot id %016lx", + FD_IP4_ADDR_FMT_ARGS( c->address ), c->dialed ? "outbound dial" : "listener", member, + c->hello.role==FD_FAILOVER_ROLE_ACTIVE ? "active" : "standby", c->hello.boot_id )); + } +} + +static void +service_candidate( fd_failover_channel_t * ch, + ulong idx, + long now, + int * busy, + uchar * payload ) { + struct candidate * c = &ch->candidates[idx]; + fd_failover_tls_budget( &c->tls ); + if( c->phase==PHASE_CONNECT ) { + if( !connect_candidate( ch, idx, now ) ) return; + } + if( c->phase==PHASE_TLS ) { + if( !handshake_candidate( ch, idx, now, busy ) ) return; + } + if( flush( ch, idx, now, busy )!=0 ) return; + if( c->phase==PHASE_HELLO ) { + if( !read_candidate_hello( ch, idx, now, busy, payload ) ) return; + } + if( c->phase==PHASE_READY && !c->tx_used ) { + pair_candidate( ch, idx, now, busy ); + } +} + +int +fd_failover_channel_poll( fd_failover_channel_t * ch, + long now, + int * busy, + ushort * type, + uchar * payload, + ulong * payload_sz ) { + int delivered = 0; + /* Established traffic gets service before the bounded accept drain. */ + if( ch->paired_idx>=0 ) { + ulong idx = (ulong)ch->paired_idx; + fd_failover_tls_budget( &ch->candidates[idx].tls ); + int flushed = flush( ch, idx, now, busy ); + if( !flushed ) { + int rc = read_frame( ch, idx, now, busy, type, payload, payload_sz ); + if( rc>0 ) { + if( *type==FD_FAILOVER_MSG_HELLO ) { lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_PROTOCOL, "it sent a second HELLO, check that both machines run the same Firedancer version" ); } + else { ch->last_rx = now; delivered = 1; } + } else if( !rc && now>=ch->last_rx && now-ch->last_rx>ch->silence_timeout ) { + lose( ch, idx, now, FD_FAILOVER_EV_TIMEOUT, LOSS_SILENCE, "the handoff connection was idle past its deadline, check the peer log" ); + } + } + if( flushed>0 && now>=ch->last_rx && now-ch->last_rx>ch->silence_timeout ) { + lose( ch, idx, now, FD_FAILOVER_EV_TIMEOUT, LOSS_SILENCE, "the handoff connection stopped making progress, check the peer log" ); + } + } + /* Expire every slot each poll, independently of round-robin service. */ + for( ulong i=0; icandidates[i]; + if( c->fd!=-1 && (int)i!=ch->paired_idx && now>=c->deadline ) { + if( c->dialed && now>=ch->dial_log_at[2] ) { + ch->dial_log_at[2] = fd_long_sat_add( now, 60000000000L ); + FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` did not finish the handshake in time, check that it runs and that the machines can reach each other on [failover.listen_port]", + FD_IP4_ADDR_FMT_ARGS( c->address ) )); + } + drop( ch, i, now, FD_FAILOVER_EV_TIMEOUT ); *busy = 1; + } + } + /* Nothing starts before we have our member certificate. Every member + drains accepts, and a member with an open handoff also dials once + the backoff ends. */ + if( FD_UNLIKELY( !ch->member_cert_set ) ) return delivered; + if( ch->listen_fd!=-1 ) accept_candidates( ch, now, busy ); + if( ch->dial_peer && ch->state==FD_FAILOVER_SESSION_BACKOFF && now>=ch->retry_at && ch->tls_ctx.ready && ch->peer_addr ) { + ulong idx = dial_slot( ch, now ); + if( FD_UNLIKELY( idx==FD_FAILOVER_CHANNEL_CANDIDATE_MAX ) ) return delivered; + int fd = socket( AF_INET, SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC, 0 ); + *busy = 1; + if( fd==-1 ) { drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); return delivered; } + ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_RETRY ); + start( ch, idx, fd, ch->peer_addr, now, PHASE_CONNECT ); + } + if( ch->paired_idx>=0 ) return delivered; + ulong serviced = 0UL; + for( ulong n=0; ncursor; + ch->cursor = (ch->cursor+1UL)%FD_FAILOVER_CHANNEL_CANDIDATE_MAX; + if( ch->candidates[idx].fd==-1 ) continue; + service_candidate( ch, idx, now, busy, payload ); + serviced++; + if( ch->paired_idx>=0 ) break; + } + return delivered; +} + +int +fd_failover_channel_send( fd_failover_channel_t * ch, + long now, + ushort type, + uchar const * payload, + ulong sz ) { + if( ch->paired_idx<0 ) return -1; + ulong idx = (ulong)ch->paired_idx; + struct candidate * c = &ch->candidates[idx]; + if( c->tx_used ) return -1; + c->tx_used = fd_failover_wire_encode( &c->wire, c->tx, type, payload, sz ); + if( !c->tx_used ) { lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_LOCAL_ENCODE, "we could not encode a frame for it" ); return -1; } + int busy = 0; + int rc = flush( ch, idx, now, &busy )<0 ? -1 : 0; + if( !rc && type==FD_FAILOVER_MSG_HANDOFF_RESULT ) ch->expected_close = 1; + return rc; +} diff --git a/src/discof/failover/fd_failover_channel.h b/src/discof/failover/fd_failover_channel.h new file mode 100644 index 00000000000..b0c7eb66819 --- /dev/null +++ b/src/discof/failover/fd_failover_channel.h @@ -0,0 +1,139 @@ +#ifndef HEADER_fd_src_discof_failover_fd_failover_channel_h +#define HEADER_fd_src_discof_failover_fd_failover_channel_h + +/* Authenticated nonblocking TCP channel for the failover pair. */ + +#include "fd_failover_wire.h" +#include "../../waltz/tls/fd_tls.h" + +#include + +#define FD_FAILOVER_CHANNEL_MAGIC (0xF17EDA2CE5FA170AUL) +#define FD_FAILOVER_CHANNEL_CANDIDATE_MAX (16UL) + +/* fd_failover_clock returns monotonic nanoseconds for every channel + deadline. */ +static inline long +fd_failover_clock( void ) { + struct timespec ts; + clock_gettime( CLOCK_MONOTONIC, &ts ); + return (long)ts.tv_sec*1000000000L + (long)ts.tv_nsec; +} + +/* Handshake, operation and retry limits in nanoseconds. */ +#define FD_FAILOVER_CHANNEL_IDLE_NANOS ( 64000000000L) +#define FD_FAILOVER_CHANNEL_BACKOFF_MIN_NANOS ( 800000000L) +#define FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS ( 12800000000L) +#define FD_FAILOVER_CHANNEL_HELLO_TIMEOUT_NANOS ( 2000000000L) + +struct fd_failover_channel; +typedef struct fd_failover_channel fd_failover_channel_t; + +FD_PROTOTYPES_BEGIN + +FD_FN_CONST ulong +fd_failover_channel_align( void ); + +FD_FN_CONST ulong +fd_failover_channel_footprint( void ); + +void * +fd_failover_channel_new( void * shmem ); + +fd_failover_channel_t * +fd_failover_channel_join( void * shch ); + +/* Opens the listener on address:port. */ +void +fd_failover_channel_init_listener( fd_failover_channel_t * channel, + uint address, + ushort port ); + +/* Starts a handoff's dial and retries at address:port. */ +void +fd_failover_channel_init_dialer( fd_failover_channel_t * channel, + uint address, + ushort port ); + +/* Configures mutual TLS and HELLO with our junk pubkey and the signer + for CertificateVerify. We call it at boot and again after a + set-identity. The private key stays in the sign tile. Returns zero + on success, -1 on invalid keys or TLS setup failure. */ +int +fd_failover_channel_set_identity( fd_failover_channel_t * channel, + uchar const junk_pubkey[ 32 ], + fd_tls_sign_t signer, + fd_failover_hello_t const * hello ); + +/* Puts our member certificate, the staked key's signature over the + cert prefix and our junk pubkey, into HELLO. We neither accept nor + dial before we have it. Returns zero on success, -1 if it does not + verify against our HELLO. */ +int +fd_failover_channel_set_member_cert( fd_failover_channel_t * channel, + uchar const * member_cert ); + +/* Closes all sockets and releases the TLS context. */ +void +fd_failover_channel_fini( fd_failover_channel_t * channel ); + +/* Returns how many connections are still in their TCP, TLS or HELLO + handshake. The paired session does not count. */ +FD_FN_PURE ulong +fd_failover_channel_pending( fd_failover_channel_t const * channel ); + +/* Updates the role in the next HELLO and leaves a paired session up, so + an open handoff can finish after the roles change. */ +void +fd_failover_channel_set_role( fd_failover_channel_t * channel, + ulong role ); + +/* Drives the channel, now is the caller's clock in nanos. Sets + *charge_busy when it made progress. Returns 1 with *out_type, + out_payload and *out_payload_sz filled when a verified post HELLO frame + arrived, else 0. out_payload must hold FD_FAILOVER_PAYLOAD_MAX bytes. */ +int +fd_failover_channel_poll( fd_failover_channel_t * channel, + long now, + int * charge_busy, + ushort * out_type, + uchar * out_payload, + ulong * out_payload_sz ); + +/* Writes one authenticated frame, now is the caller's clock in nanos. + Returns 0 on success, -1 when the channel is not paired, already has a + pending frame, or encoding or writing fails. Encoding and write + failures tear down the session. */ +int +fd_failover_channel_send( fd_failover_channel_t * channel, + long now, + ushort type, + uchar const * payload, + ulong payload_sz ); + +/* Accessors. */ +FD_FN_PURE ulong fd_failover_channel_state ( fd_failover_channel_t const * channel ); +FD_FN_PURE int fd_failover_channel_listen_fd ( fd_failover_channel_t const * channel ); +FD_FN_PURE int fd_failover_channel_tx_pending( fd_failover_channel_t const * channel ); +FD_FN_PURE fd_failover_hello_t const * fd_failover_channel_peer_hello( fd_failover_channel_t const * channel ); + +/* Changes whenever a new authenticated session replaces the previous one. */ +FD_FN_PURE ulong fd_failover_channel_generation( fd_failover_channel_t const * channel ); + +/* Reads the bound port back from the listen socket, for binds to port zero. */ +ushort +fd_failover_channel_listen_port( fd_failover_channel_t const * channel ); + +/* Closes any live connection and returns the channel to its resting state. */ +void +fd_failover_channel_hangup( fd_failover_channel_t * channel, + long now ); + +/* Drops a paired session after an authenticated protocol violation. */ +void +fd_failover_channel_protocol_error( fd_failover_channel_t * channel, + long now ); + +FD_PROTOTYPES_END + +#endif /* HEADER_fd_src_discof_failover_fd_failover_channel_h */ diff --git a/src/discof/failover/fd_failover_tls.c b/src/discof/failover/fd_failover_tls.c new file mode 100644 index 00000000000..0777a60e731 --- /dev/null +++ b/src/discof/failover/fd_failover_tls.c @@ -0,0 +1,173 @@ +#include "fd_failover_tls.h" + +#include "../../ballet/ed25519/fd_x25519.h" +#include "../../ballet/x509/fd_x509_mock.h" + +#include + +/* ALPN id, TLS length-prefixed */ +static uchar const ALPN[] = "\x17" "fd-failover/on-demand/1"; + +FD_STATIC_ASSERT( sizeof(ALPN)==25UL, alpn_size ); + +int +fd_failover_tls_ctx_init( fd_failover_tls_ctx_t * ctx, + uchar const public_key[ 32 ], + fd_tls_sign_t signer ) { + fd_failover_tls_ctx_fini( ctx ); + + uchar seed[ 32 ]; + if( FD_UNLIKELY( !fd_rng_secure( seed, 32UL ) ) ) return -1; + fd_chacha_rng_t * rng = fd_chacha_rng_join( fd_chacha_rng_new( ctx->rng, FD_CHACHA_RNG_MODE_MOD ) ); + FD_TEST( rng ); + fd_chacha_rng_init( rng, seed, FD_CHACHA_RNG_ALGO_CHACHA20 ); + fd_memzero_explicit( seed, sizeof(seed) ); + + fd_tls_t * tls = fd_tls_join( fd_tls_new( &ctx->tls ) ); + FD_TEST( tls ); + tls->rng = rng; + tls->sign = signer; + fd_memcpy( tls->cert_public_key, public_key, 32UL ); + fd_x509_mock_cert( tls->cert_x509, tls->cert_public_key ); + tls->cert_x509_sz = FD_X509_MOCK_CERT_SZ; + fd_memcpy( tls->alpn, ALPN, sizeof(ALPN)-1UL ); + tls->alpn_sz = sizeof(ALPN)-1UL; + tls->quic = 0; + /* Both members present a certificate, the listener checks the dialer's. */ + tls->req_client_cert = 1; + + ctx->ready = 1; + return 0; +} + +void +fd_failover_tls_ctx_fini( fd_failover_tls_ctx_t * ctx ) { + fd_memzero_explicit( ctx, sizeof(*ctx) ); +} + +int +fd_failover_tls_new( fd_failover_tls_t * tls, + fd_failover_tls_ctx_t * ctx, + int fd, + int dial_peer ) { + if( FD_UNLIKELY( !ctx->ready ) ) return -1; + fd_memset( tls, 0, sizeof(*tls) ); + tls->fd = fd; + fd_tlsrec_conn_init( &tls->conn, &ctx->tls, !dial_peer ); + /* Fresh X25519 key share per connection, straight from the kernel. */ + uchar seed[ 32 ]; + if( FD_UNLIKELY( !fd_rng_secure( seed, 32UL ) ) ) return -1; + fd_chacha_rng_init( ctx->rng, seed, FD_CHACHA_RNG_ALGO_CHACHA20 ); + fd_memzero_explicit( seed, sizeof(seed) ); + if( FD_UNLIKELY( !fd_rng_secure( tls->conn.tls.kex_private_key, 32UL ) ) ) return -1; + fd_x25519_public( tls->conn.tls.kex_public_key, tls->conn.tls.kex_private_key ); + fd_tlsrec_sock_init( &tls->sock ); + fd_failover_tls_budget( tls ); + return 0; +} + +void +fd_failover_tls_fini( fd_failover_tls_t * tls ) { + if( FD_LIKELY( tls->fd!=-1 && fd_tlsrec_conn_is_ready( &tls->conn ) && !tls->conn.tx_closed ) ) { + (void)fd_tlsrec_sock_close( &tls->sock, &tls->conn, tls->fd ); + } + fd_memzero_explicit( &tls->conn, sizeof(tls->conn) ); + fd_memzero_explicit( &tls->sock, sizeof(tls->sock) ); + tls->fd = -1; + tls->verified = 0; + tls->paired = 0; + tls->received = 0UL; +} + +void +fd_failover_tls_budget( fd_failover_tls_t * tls ) { + tls->read_budget = 1; + tls->write_budget = 1; +} + +/* pull reads one batch of ciphertext into the record layer. Returns 0 + (maybe nothing read) or -1 on error or peer close. */ + +static int +pull( fd_failover_tls_t * tls ) { + if( FD_UNLIKELY( !tls->read_budget ) ) return 0; + tls->read_budget--; + ulong rx_sz = 0UL; + int rc = fd_tlsrec_sock_rx( &tls->sock, &tls->conn, tls->fd, &rx_sz ); + if( FD_UNLIKELY( rc ) ) { + tls->peer_closed = rc==FD_TLSREC_SOCK_ERR_EOF; + return -1; + } + if( FD_LIKELY( !tls->paired ) ) { + tls->received += rx_sz; + if( FD_UNLIKELY( tls->received>FD_FAILOVER_TLS_PREPAIR_MAX ) ) return -1; + } + return 0; +} + +int +fd_failover_tls_handshake( fd_failover_tls_t * tls ) { + if( FD_LIKELY( tls->verified ) ) return 1; + if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) ) ) { + FD_LOG_DEBUG(( "failover TLS handshake failed: %s", fd_tls_reason_cstr( tls->conn.hs.base.reason ) )); + return -1; + } + if( FD_UNLIKELY( fd_tlsrec_sock_flush( &tls->sock, tls->fd )<0 ) ) return -1; + if( FD_LIKELY( !fd_tlsrec_conn_is_ready( &tls->conn ) ) ) { + if( FD_UNLIKELY( pull( tls ) ) ) return -1; + if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) ) ) { + FD_LOG_DEBUG(( "failover TLS handshake failed: %s", fd_tls_reason_cstr( tls->conn.hs.base.reason ) )); + return -1; + } + if( FD_LIKELY( !fd_tlsrec_conn_is_ready( &tls->conn ) ) ) return 0; + } + + /* Handshake done, the peer signed with some key and agreed on our ALPN. + We keep that key, the channel checks it against HELLO. */ + + if( !fd_tlsrec_conn_is_server( &tls->conn ) ) { + fd_tls_estate_cli_t const * cli = &tls->conn.hs.cli; + if( FD_UNLIKELY( cli->server_key_type!=FD_TLS_KEY_ED25519 || + cli->server_pubkey_len!=32UL || !cli->alpn_negotiated ) ) return -1; + fd_memcpy( tls->peer_pubkey, cli->server_pubkey, 32UL ); + } else { + /* fd_tls only gets here with a client cert and a CertificateVerify + it checked against client_pubkey. */ + fd_tls_estate_srv_t const * srv = &tls->conn.hs.srv; + fd_memcpy( tls->peer_pubkey, srv->client_pubkey, 32UL ); + } + tls->verified = 1; + return 1; +} + +long +fd_failover_tls_read( fd_failover_tls_t * tls, + void * buf, + ulong sz ) { + if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) ) ) return -1L; + if( FD_LIKELY( !fd_tlsrec_sock_rx_avail( &tls->sock ) ) ) { + if( FD_UNLIKELY( tls->conn.rx_closed ) ) { + tls->peer_closed = 1; + return -1L; + } + if( FD_UNLIKELY( fd_tlsrec_sock_flush( &tls->sock, tls->fd )<0 ) ) return -1L; + if( FD_UNLIKELY( pull( tls ) ) ) return -1L; + } + return (long)fd_tlsrec_sock_rx_pop( &tls->sock, buf, sz ); +} + +long +fd_failover_tls_write( fd_failover_tls_t * tls, + void const * buf, + ulong sz ) { + if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) || tls->conn.tx_closed ) ) return -1L; + if( FD_UNLIKELY( fd_tlsrec_sock_tx_pending( &tls->sock ) ) ) { + int rc = fd_tlsrec_sock_flush( &tls->sock, tls->fd ); + if( rc ) return rc<0 ? -1L : 0L; + } + if( FD_UNLIKELY( !tls->write_budget ) ) return 0L; + tls->write_budget--; + ulong consumed = 0UL; + if( FD_UNLIKELY( fd_tlsrec_sock_tx( &tls->sock, &tls->conn, tls->fd, buf, sz, &consumed ) ) ) return -1L; + return (long)consumed; +} diff --git a/src/discof/failover/fd_failover_tls.h b/src/discof/failover/fd_failover_tls.h new file mode 100644 index 00000000000..cca4bef6122 --- /dev/null +++ b/src/discof/failover/fd_failover_tls.h @@ -0,0 +1,103 @@ +#ifndef HEADER_fd_src_discof_failover_fd_failover_tls_h +#define HEADER_fd_src_discof_failover_fd_failover_tls_h + +/* TLS 1.3 over TCP for one failover candidate, built on fd_tls and + fd_tlsrec. Both ends present an Ed25519 cert for their junk identity. + TLS does not check the peer key against an expected key, the channel + checks it against HELLO and its member certificate. The caller owns + the socket. */ + +#include "../../waltz/tlsrec/fd_tlsrec_sock.h" +#include "../../ballet/chacha/fd_chacha_rng.h" + +/* Max ciphertext accepted from a candidate before it pairs */ +#define FD_FAILOVER_TLS_PREPAIR_MAX (32768UL) + +/* Per-channel TLS config, the fd_tls template and its RNG. The sign + tile signs our CertificateVerify, no private key is kept here. */ + +struct fd_failover_tls_ctx { + fd_tls_t tls; + fd_chacha_rng_t rng[ 1 ]; + int ready; +}; +typedef struct fd_failover_tls_ctx fd_failover_tls_ctx_t; + +/* One TLS connection. */ + +struct fd_failover_tls { + fd_tlsrec_conn_t conn; + fd_tlsrec_sock_t sock; + int fd; + int verified; /* handshake done, peer_pubkey is set */ + int paired; /* set by the channel after HELLO */ + int peer_closed; /* orderly EOF or close_notify */ + uchar peer_pubkey[ 32 ]; /* junk key the peer signed the handshake with */ + int read_budget; /* socket reads left this turn */ + int write_budget; /* record writes left this turn */ + ulong received; /* ciphertext bytes read before pairing */ +}; +typedef struct fd_failover_tls fd_failover_tls_t; + +FD_PROTOTYPES_BEGIN + +/* fd_failover_tls_ctx_init sets our junk pubkey and the signer for + CertificateVerify and seeds the RNG. It and every new connection read + entropy from getrandom, which the sandbox allows. Returns 0 on + success, -1 if entropy fails. */ + +int +fd_failover_tls_ctx_init( fd_failover_tls_ctx_t * ctx, + uchar const public_key[ 32 ], + fd_tls_sign_t signer ); + +/* fd_failover_tls_ctx_fini wipes the context. */ + +void +fd_failover_tls_ctx_fini( fd_failover_tls_ctx_t * ctx ); + +/* fd_failover_tls_new binds a fresh connection to fd. The listener + requires a client cert. Returns 0 on success, -1 if ctx is not + initialized or entropy fails. */ + +int +fd_failover_tls_new( fd_failover_tls_t * tls, + fd_failover_tls_ctx_t * ctx, + int fd, + int dial_peer ); + +/* fd_failover_tls_fini sends close_notify if the connection is up and + wipes the connection state. The caller closes fd. */ + +void +fd_failover_tls_fini( fd_failover_tls_t * tls ); + +/* fd_failover_tls_budget allows one socket read and one record write + per service turn. */ + +void +fd_failover_tls_budget( fd_failover_tls_t * tls ); + +/* fd_failover_tls_handshake returns 1 once the handshake is done and + the peer signed it with an Ed25519 key, which is then in peer_pubkey, + 0 while pending, -1 on failure. fd_failover_tls_read and + fd_failover_tls_write return bytes moved, 0 when pending or out of + budget, -1 on error or peer close. A write may take fewer bytes than + offered (at most one record), the caller retries the rest. */ + +int +fd_failover_tls_handshake( fd_failover_tls_t * tls ); + +long +fd_failover_tls_read( fd_failover_tls_t * tls, + void * buf, + ulong sz ); + +long +fd_failover_tls_write( fd_failover_tls_t * tls, + void const * buf, + ulong sz ); + +FD_PROTOTYPES_END + +#endif /* HEADER_fd_src_discof_failover_fd_failover_tls_h */ diff --git a/src/discof/failover/test_failover_channel.c b/src/discof/failover/test_failover_channel.c new file mode 100644 index 00000000000..97580598401 --- /dev/null +++ b/src/discof/failover/test_failover_channel.c @@ -0,0 +1,106 @@ +#include "fd_failover_channel.h" +#include "../../disco/keyguard/fd_keyguard.h" +#include "../../ballet/ed25519/fd_ed25519.h" +#include "../../util/net/fd_ip4.h" + +static uchar scratch[ 2 ][ 8UL<<20 ] __attribute__((aligned(128))); +static uchar keys[ 3 ][ 64 ] = { { 1 }, { 2 }, { 9 } }; +static fd_sha512_t sha[ 1 ]; + +/* Stands in for the sign tile, the member's junk key signs its TLS. */ +static void +sign_fn( void * ctx, + uchar sig[ static FD_ED25519_SIG_SZ ], + uchar const payload[ static FD_TLS_CV_SIGN_SZ ] ) { + uchar const * key = ctx; + fd_ed25519_sign( sig, payload, FD_TLS_CV_SIGN_SZ, key+32UL, key, sha ); +} + +static fd_failover_channel_t * +member( ulong idx, ulong staked_idx ) { + fd_failover_channel_t * ch = fd_failover_channel_join( fd_failover_channel_new( scratch[idx] ) ); + FD_TEST( ch ); + fd_failover_hello_t hello = { .version=FD_FAILOVER_VERSION, .boot_id=idx+1UL, + .role=(uchar)(idx ? FD_FAILOVER_ROLE_STANDBY : FD_FAILOVER_ROLE_ACTIVE), + .mode=FD_FAILOVER_MODE_TOWER }; + fd_memcpy( hello.junk_pubkey, keys[idx]+32UL, 32UL ); + fd_memcpy( hello.staked_pubkey, keys[staked_idx]+32UL, 32UL ); + fd_memset( hello.vote_account, 0xBB, 32UL ); + FD_TEST( !fd_failover_channel_set_identity( ch, keys[idx]+32UL, (fd_tls_sign_t){ .ctx=keys[idx], .sign_fn=sign_fn }, &hello ) ); + uchar msg[ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ], cert[ 64 ]; + fd_failover_member_cert_msg( msg, keys[idx]+32UL ); + fd_ed25519_sign( cert, msg, sizeof(msg), keys[staked_idx]+32UL, keys[staked_idx], sha ); + FD_TEST( !fd_failover_channel_set_member_cert( ch, cert ) ); + fd_failover_channel_init_listener( ch, FD_IP4_ADDR(127,0,0,1), 0 ); + return ch; +} + +static int +poll_one( fd_failover_channel_t * ch, ushort * type, uchar * payload, ulong * sz ) { + int busy = 0; + return fd_failover_channel_poll( ch, fd_failover_clock(), &busy, type, payload, sz ); +} + +static void +test_pair_and_transfer( void ) { + fd_failover_channel_t * a = member( 0UL, 2UL ); + fd_failover_channel_t * b = member( 1UL, 2UL ); + fd_failover_channel_init_dialer( b, FD_IP4_ADDR(127,0,0,1), fd_failover_channel_listen_port( a ) ); + long until = fd_failover_clock()+5000000000L; + ushort type; + ulong sz; + uchar payload[ FD_FAILOVER_PAYLOAD_MAX ]; + while( fd_failover_channel_state( a )!=FD_FAILOVER_SESSION_PAIRED || + fd_failover_channel_state( b )!=FD_FAILOVER_SESSION_PAIRED ) { + FD_TEST( fd_failover_clock()boot_id && !fd_failover_channel_peer_hello( b )->boot_id ); + fd_failover_channel_fini( a ); + fd_failover_channel_fini( b ); +} + +int +main( int argc, char ** argv ) { + fd_boot( &argc, &argv ); + FD_TEST( fd_failover_channel_footprint()<=sizeof(scratch[0]) ); + FD_TEST( fd_sha512_join( fd_sha512_new( sha ) ) ); + for( ulong i=0UL; i<3UL; i++ ) fd_ed25519_public_from_private( keys[i]+32UL, keys[i], sha ); + test_pair_and_transfer(); + test_wrong_staked_key(); + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} From 89a11396bc3b9ec779038dd85786a7e2d7d6f05a Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Fri, 18 Sep 2026 17:21:51 +0000 Subject: [PATCH 08/23] failover: add identity switching and failover commands to the admin bus --- src/disco/events/generated/fd_event_gen.h | 1 + src/disco/events/schema/admin_command.json | 3 +- src/disco/events/schema/events.proto | 1 + src/discof/admin/fd_admin_tile.c | 376 ++++++++++++++++++++- src/discof/admin/fd_adminctl.h | 130 +++++++ src/discof/failover/fd_failover_bus.h | 63 ++++ 6 files changed, 572 insertions(+), 2 deletions(-) create mode 100644 src/discof/failover/fd_failover_bus.h diff --git a/src/disco/events/generated/fd_event_gen.h b/src/disco/events/generated/fd_event_gen.h index 3eaefd8d29d..9053ce9e663 100644 --- a/src/disco/events/generated/fd_event_gen.h +++ b/src/disco/events/generated/fd_event_gen.h @@ -517,6 +517,7 @@ typedef struct fd_event_snapshot_created fd_event_snapshot_created_t; #define FD_EVENT_ADMIN_COMMAND_TYPE_ADD_AUTHORIZED_VOTER (3) /* Add an authorized voter key. */ #define FD_EVENT_ADMIN_COMMAND_TYPE_REMOVE_ALL_AUTHORIZED_VOTERS (4) /* Remove every authorized voter key. */ #define FD_EVENT_ADMIN_COMMAND_TYPE_SNAPSHOT_CREATE (5) /* Request creation of a snapshot. */ +#define FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER (6) /* Run a failover command. */ /* The result of processing the admin command. */ #define FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS (1) /* The command completed successfully. */ diff --git a/src/disco/events/schema/admin_command.json b/src/disco/events/schema/admin_command.json index 3f478e037e4..01fb9e8ef12 100644 --- a/src/disco/events/schema/admin_command.json +++ b/src/disco/events/schema/admin_command.json @@ -11,7 +11,8 @@ "get_identity": { "description": "Read the running validator identity." }, "add_authorized_voter": { "description": "Add an authorized voter key." }, "remove_all_authorized_voters": { "description": "Remove every authorized voter key." }, - "snapshot_create": { "description": "Request creation of a snapshot." } + "snapshot_create": { "description": "Request creation of a snapshot." }, + "failover": { "description": "Run a failover command." } } }, "result": { diff --git a/src/disco/events/schema/events.proto b/src/disco/events/schema/events.proto index ae243783384..fcad7278760 100644 --- a/src/disco/events/schema/events.proto +++ b/src/disco/events/schema/events.proto @@ -405,6 +405,7 @@ enum AdminCommandType { ADMIN_COMMAND_TYPE_ADD_AUTHORIZED_VOTER = 3; // Add an authorized voter key. ADMIN_COMMAND_TYPE_REMOVE_ALL_AUTHORIZED_VOTERS = 4; // Remove every authorized voter key. ADMIN_COMMAND_TYPE_SNAPSHOT_CREATE = 5; // Request creation of a snapshot. + ADMIN_COMMAND_TYPE_FAILOVER = 6; // Run a failover command. } // The result of processing the admin command. diff --git a/src/discof/admin/fd_admin_tile.c b/src/discof/admin/fd_admin_tile.c index a4af9f8940f..77d09032852 100644 --- a/src/discof/admin/fd_admin_tile.c +++ b/src/discof/admin/fd_admin_tile.c @@ -4,6 +4,7 @@ #include "../../disco/keyguard/fd_keyload.h" #include "fd_adminctl.h" +#include "../failover/fd_failover_bus.h" #include #include "generated/fd_admin_tile_seccomp.h" @@ -25,6 +26,29 @@ struct fd_admin_tile_ctx { long idle_poll_ticks; long deadline_ticks; /* next adminctl poll */ int failover_enabled; /* failover moves the identity, set-identity is refused */ + + /* Failover commands go to the failover tile over the bus, one at a + time like snapshot creation. */ + ulong failov_out_idx; /* admin_failov stem out index */ + fd_wksp_t * failov_out_mem; + ulong failov_out_chunk0; + ulong failov_out_wmark; + ulong failov_out_chunk; + ulong failov_in_idx; /* failov_admin stem in index */ + ulong in_cnt; /* stem ins, failov_admin is found among them */ + fd_wksp_t * failov_in_mem; + ulong failov_in_chunk0; + ulong failov_in_wmark; + fd_failover_bus_msg_t failov_in; /* frame copied in during_frag */ + ulong failover_slot_idx; /* adminctl slot parked on the bus, or ULONG_MAX */ + ulong failover_slot_cmd; /* FD_ADMINCTL_FAILOVER_CMD_* */ + ulong failover_nonce; /* nonce of the last command forwarded to the failover tile */ + ulong failover_answered; /* nonce of the last command the failover tile responded to */ + ulong failover_start_time; /* command start retained until the failover tile responds */ + long failover_deadline; /* tickcount after which we report unresponsive */ + char failover_args_json[ 96 ]; /* the parked command and its flags for the event */ + ulong failover_args_json_len; + char failover_cmd_cstr[ 48 ]; /* the parked command and its flags for the log, empty for status */ }; typedef struct fd_admin_tile_ctx fd_admin_tile_ctx_t; @@ -102,6 +126,7 @@ unprivileged_init( fd_topo_t const * topo, ctx->idle_poll_ticks = (long)((double)FD_SLEEP_PARK_CAP_NS*fd_tempo_tick_per_ns( NULL )); ctx->deadline_ticks = 0L; ctx->failover_enabled = fd_topo_find_tile( topo, "failov", 0UL )!=ULONG_MAX; + ctx->failover_slot_idx = ULONG_MAX; ctx->topo = topo; fd_topo_obj_t const * adminctl_obj = fd_topo_find_tile_obj( topo, tile, "adminctl" ); @@ -112,8 +137,33 @@ unprivileged_init( fd_topo_t const * topo, ctx->replay_out_idx = fd_topo_find_tile_out_link( topo, tile, "admin_replay", 0UL ); + /* The failover bus links exist only when the failover tile does. */ + ctx->failov_out_idx = fd_topo_find_tile_out_link( topo, tile, "admin_failov", 0UL ); + ctx->failov_in_idx = fd_topo_find_tile_in_link ( topo, tile, "failov_admin", 0UL ); + ctx->in_cnt = tile->in_cnt; + if( FD_UNLIKELY( (ctx->failov_out_idx==ULONG_MAX)!=(ctx->failov_in_idx==ULONG_MAX) ) ) { + FD_LOG_ERR(( "admin tile needs both the admin_failov and failov_admin links or neither" )); + } + if( FD_UNLIKELY( ctx->failov_out_idx!=ULONG_MAX ) ) { + fd_topo_link_t const * out_link = &topo->links[ tile->out_link_id[ ctx->failov_out_idx ] ]; + ctx->failov_out_mem = topo->workspaces[ topo->objs[ out_link->dcache_obj_id ].wksp_id ].wksp; + ctx->failov_out_chunk0 = fd_dcache_compact_chunk0( ctx->failov_out_mem, out_link->dcache ); + ctx->failov_out_wmark = fd_dcache_compact_wmark ( ctx->failov_out_mem, out_link->dcache, out_link->mtu ); + ctx->failov_out_chunk = ctx->failov_out_chunk0; + + fd_topo_link_t const * in_link = &topo->links[ tile->in_link_id[ ctx->failov_in_idx ] ]; + ctx->failov_in_mem = topo->workspaces[ topo->objs[ in_link->dcache_obj_id ].wksp_id ].wksp; + ctx->failov_in_chunk0 = fd_dcache_compact_chunk0( ctx->failov_in_mem, in_link->dcache ); + ctx->failov_in_wmark = fd_dcache_compact_wmark ( ctx->failov_in_mem, in_link->dcache, in_link->mtu ); + } + for( ulong i=0UL; iin_cnt; i++ ) { fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ]; + /* The stem numbers only the in links it polls, and we compare its + numbers with topology in link indexes, so under failover every in + link is polled. */ + if( FD_UNLIKELY( ctx->failov_in_idx!=ULONG_MAX ) ) FD_TEST( tile->in_link_poll[ i ] ); + if( FD_UNLIKELY( i==ctx->failov_in_idx ) ) continue; if( FD_UNLIKELY( strcmp( link->name, "replay_admin" ) ) ) { FD_LOG_ERR(( "unexpected input link name %s", link->name )); } @@ -423,6 +473,13 @@ poll_set_identity( fd_admin_tile_ctx_t * ctx, if( strcmp( tile->name, "sign" ) ) continue; fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id ); memcpy( sign->bytes, keypair, 64UL ); + if( FD_UNLIKELY( ctx->failover_enabled ) ) { + /* Under failover the sign tile selects a keypair it loaded at + boot by its public key. */ + sign->param = FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY; + fd_memcpy( sign->bytes, keypair+32UL, 32UL ); + fd_memset( sign->bytes+32UL, 0, 32UL ); + } FD_COMPILER_MFENCE(); sign->state = FD_KEYSWITCH_STATE_SWITCH_PENDING; FD_COMPILER_MFENCE(); @@ -1181,6 +1238,275 @@ next_deadline( fd_admin_tile_ctx_t const * ctx ) { return ctx->deadline_ticks; } +/* The failover tile responds to commands. We check the + ABI, forward the command over the bus and park the adminctl slot + until the response comes back or the deadline passes, like snapshot + creation waits on replay. */ + +static fd_event_admin_command_t * +failover_event_args( fd_admin_tile_ctx_t const * ctx, + fd_event_admin_command_t * event ) { + if( FD_LIKELY( ctx->failover_args_json_len ) ) { + fd_memcpy( event->args_json, ctx->failover_args_json, ctx->failover_args_json_len ); + event->args_json_len = ctx->failover_args_json_len; + } + return event; +} + +/* failover_result_name is the event's custom result for a failover + command that did not succeed. */ +static char const * +failover_result_name( ulong result ) { + switch( result ) { + case FD_FAILOVER_CONTROL_RESULT_BUSY: return "busy"; + case FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE: return "unresponsive"; + case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: return "bad_role"; + case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: return "in_progress"; + case FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY: return "peer_unready"; + case FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED: return "peer_unverified"; + case FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE: return "peer_active"; + case FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING: return "handoff_pending"; + case FD_FAILOVER_CONTROL_RESULT_TAKEN: return "taken"; + case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "staked_seen"; + case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER: return "no_final_tower"; + case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: return "replay_behind"; + case FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS: return "no_active_address"; + default: return "refused"; + } +} + +static void +failover_complete( fd_admin_tile_ctx_t * ctx, + ulong result, + void const * resp, + ulong resp_sz ) { + fd_event_admin_command_t event = { + .type = FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER, + .args_json = { '{', '}' }, + .args_json_len = 2UL, + .start_time = ctx->failover_start_time, + .payload_version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, + .has_payload_version = 1, + .payload_size = sizeof(fd_adminctl_failover_req_t), + }; + switch( result ) { + case FD_ADMINCTL_RESULT_SUCCESS: + report_admin_command( failover_event_args( ctx, &event ), FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS ); + break; + default: + report_admin_command_custom_result( failover_event_args( ctx, &event ), failover_result_name( result ) ); + break; + } + fd_adminctl_complete_response( ctx->adminctl, ctx->failover_slot_idx, result, resp, resp_sz ); + if( FD_LIKELY( ctx->failover_cmd_cstr[ 0 ] ) ) { + if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) { + FD_LOG_NOTICE(( "`failover %s` accepted by the failover tile, follow it with `failover status`", ctx->failover_cmd_cstr )); + } else if( FD_UNLIKELY( result!=FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE ) ) { + FD_LOG_WARNING(( "`failover %s` refused by the failover tile (%s), nothing was done", ctx->failover_cmd_cstr, failover_result_name( result ) )); + } + } + ctx->failover_slot_idx = ULONG_MAX; + ctx->failover_start_time = 0UL; + ctx->failover_deadline = 0L; + ctx->failover_args_json_len = 0UL; + ctx->failover_cmd_cstr[ 0 ] = '\0'; +} + +static void +failover_request( fd_admin_tile_ctx_t * ctx, + fd_stem_context_t * stem, + ulong slot_idx, + void const * data, + ulong data_sz ) { + + fd_adminctl_t * adminctl = ctx->adminctl; + fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER, data, data_sz ); + + if( FD_UNLIKELY( data_sz=FD_ADMINCTL_FAILOVER_CMD_CNT ) invalid = "unknown command"; + else if( req.flags & ~(FD_ADMINCTL_FAILOVER_FLAG_YES|FD_ADMINCTL_FAILOVER_FLAG_FORCE) ) invalid = "unknown flags"; + else if( (req.cmd==FD_ADMINCTL_FAILOVER_CMD_PROMOTE)!=force ) invalid = "--force goes with promote, and a promote without it is sent as a handoff"; + else if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS && req.flags ) invalid = "status takes no flags"; + else if( req.cmd!=FD_ADMINCTL_FAILOVER_CMD_HANDOFF && (req.addr || req.port) ) invalid = "an address or port goes only with a handoff"; + else if( req.reserved[ 0 ] || req.reserved[ 1 ] ) invalid = "reserved bytes are set"; + if( FD_UNLIKELY( invalid ) ) { + FD_LOG_WARNING(( "refusing adminctl failover cmd %lu flags %lu, %s", req.cmd, req.flags, invalid )); + report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNKNOWN_COMMAND ); + fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ); + return; + } + int status = req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS; + if( status ) { + FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, + "{\"command\":\"status\"}" ) ); + } else { + FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, + "{\"command\":\"%s\",\"yes\":%s,\"force\":%s}", fd_adminctl_failover_cmd_name( req.cmd ), + (req.flags & FD_ADMINCTL_FAILOVER_FLAG_YES) ? "true" : "false", + force ? "true" : "false" ) ); + } + char cmd_cstr[ 48 ]; + FD_TEST( fd_cstr_printf_check( cmd_cstr, sizeof(cmd_cstr), NULL, "%s%s%s", fd_adminctl_failover_cmd_typed( req.cmd ), + (req.flags & FD_ADMINCTL_FAILOVER_FLAG_YES) ? " --yes" : "", + force ? " --force" : "" ) ); + + if( FD_UNLIKELY( ctx->failov_out_idx==ULONG_MAX ) ) { + FD_LOG_WARNING(( "`failover %s` refused, the admin tile has no failover bus link", cmd_cstr )); + report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNSUPPORTED ); + fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNSUPPORTED ); + return; + } + + /* One command at a time. A command stays outstanding until the + failover tile responds, even after its deadline, so commands cannot + pile up behind a stalled failover tile. Status then reports the + failover tile unresponsive rather than busy. */ + if( FD_UNLIKELY( ctx->failover_answered!=ctx->failover_nonce ) ) { + if( FD_UNLIKELY( status && ctx->failover_slot_idx==ULONG_MAX ) ) { + FD_LOG_WARNING(( "`failover status` refused, the failover tile has not responded to an earlier command, check the log" )); + report_admin_command_custom_result( &event, "unresponsive" ); + fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE ); + return; + } + FD_LOG_WARNING(( "`failover %s` refused, another failover command is still waiting on the failover tile, try again", cmd_cstr )); + report_admin_command_custom_result( &event, "busy" ); + fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_BUSY ); + return; + } + + /* Kept for the event and the log of the response. */ + FD_TEST( event.args_json_len<=sizeof(ctx->failover_args_json) ); + fd_memcpy( ctx->failover_args_json, event.args_json, event.args_json_len ); + ctx->failover_args_json_len = event.args_json_len; + if( status ) ctx->failover_cmd_cstr[ 0 ] = '\0'; + else { + fd_cstr_ncpy( ctx->failover_cmd_cstr, cmd_cstr, sizeof(ctx->failover_cmd_cstr) ); + FD_LOG_NOTICE(( "forwarding `failover %s` to the failover tile", cmd_cstr )); + } + + fd_failover_bus_msg_t * msg = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk ); + fd_memset( msg, 0, sizeof(*msg) ); + msg->nonce = ++ctx->failover_nonce; + fd_memcpy( msg->payload, &req, sizeof(req) ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_REQUEST, ctx->failov_out_chunk, sizeof(*msg), 0UL, tspub, tspub ); + ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*msg), ctx->failov_out_chunk0, ctx->failov_out_wmark ); + + ctx->failover_slot_idx = slot_idx; + ctx->failover_slot_cmd = req.cmd; + ctx->failover_start_time = event.start_time; + ctx->failover_deadline = fd_tickcount() + (long)( (double)FD_FAILOVER_BUS_DEADLINE_NANOS*fd_tempo_tick_per_ns( NULL ) ); +} + +static void +failover_response( fd_admin_tile_ctx_t * ctx ) { + fd_failover_bus_msg_t const * msg = &ctx->failov_in; + /* Even a late response means the last command is no longer outstanding. */ + if( FD_LIKELY( msg->nonce==ctx->failover_nonce ) ) ctx->failover_answered = msg->nonce; + /* A response for an older command, or one after the deadline already + answered the operator, is dropped. */ + if( FD_UNLIKELY( ctx->failover_slot_idx==ULONG_MAX || msg->nonce!=ctx->failover_nonce ) ) { + FD_LOG_WARNING(( "dropping a stale failover command response" )); + return; + } + /* Status and command responses each have their own payload. */ + ulong resp_sz = ctx->failover_slot_cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? sizeof(fd_adminctl_failover_status_resp_t) + : sizeof(fd_adminctl_failover_control_resp_t); + failover_complete( ctx, msg->result, msg->payload, resp_sz ); +} + +/* The failover tile asked for an identity switch. We only pass the + public key on, the sign tile selects the keypair it loaded at boot. + Like set-identity this blocks until every tile has switched. We poll + no adminctl command meanwhile, so `failover status` and every failover + command wait for it. */ + +static void +failover_switch_request( fd_admin_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + ulong nonce = ctx->failov_in.nonce; + fd_failover_switch_req_t req; + fd_memcpy( &req, ctx->failov_in.payload, sizeof(req) ); + + fd_failover_switch_resp_t response; + fd_memset( &response, 0, sizeof(response) ); + response.result = FD_FAILOVER_SWITCH_ERR_DISABLED; + if( FD_LIKELY( ctx->failover_enabled ) ) { + fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_SET_IDENTITY, NULL, 0UL ); + FD_BASE58_ENCODE_32_BYTES( ctx->identity_pubkey, old_identity ); + FD_BASE58_ENCODE_32_BYTES( req.identity, new_identity ); + FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, + "{\"old_identity\":\"%s\",\"identity\":\"%s\",\"source\":\"failover\"}", old_identity, new_identity ) ); + FD_LOG_NOTICE(( "failover: switching the identity from `%s` to `%s`, failover commands wait until every tile has switched", old_identity, new_identity )); + + /* No private half, only the public key reaches the keyswitches. */ + uchar keypair[ 64 ]; + fd_memset( keypair, 0, 32UL ); + fd_memcpy( keypair+32UL, req.identity, 32UL ); + + ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; + ulong identity_outset = (ulong)fd_log_wallclock(); + for(;;) { + if( FD_UNLIKELY( poll_set_identity( ctx, &state, identity_outset, keypair, NULL, 0UL ) ) ) break; + } + fd_memcpy( ctx->identity_pubkey, req.identity, 32UL ); + + /* The watermark is the tower's output sequence at its halt, which + the tower tile leaves in its keyswitch result. */ + response.result = FD_FAILOVER_SWITCH_OK; + response.tower_watermark = find_identity_keyswitch( ctx, "tower" )->result; + report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS ); + FD_LOG_NOTICE(( "failover: every tile switched to `%s`, the tower halted signing at sequence %lu", new_identity, response.tower_watermark )); + } else { + FD_LOG_WARNING(( "the failover tile asked for an identity switch but failover is off, refusing" )); + } + + fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk ); + fd_memset( out, 0, sizeof(*out) ); + out->nonce = nonce; + fd_memcpy( out->payload, &response, sizeof(response) ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_SWITCH_RESP, ctx->failov_out_chunk, sizeof(*out), 0UL, tspub, tspub ); + ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*out), ctx->failov_out_chunk0, ctx->failov_out_wmark ); +} + +/* Returns whether a frag is waiting on failov_admin. */ + +static int +failov_frag_waiting( fd_admin_tile_ctx_t const * ctx, + fd_stem_context_t const * stem ) { + for( ulong i=0UL; iin_cnt; i++ ) { + fd_stem_tile_in_t const * in = &stem->in[ i ]; + if( FD_LIKELY( (ulong)in->idx==ctx->failov_in_idx ) ) return fd_seq_diff( in->seq, fd_frag_meta_seq_query( in->mline ) )<=0L; + } + return 0; +} + static inline void FD_FN_SENSITIVE after_credit( fd_admin_tile_ctx_t * ctx, fd_stem_context_t * stem, @@ -1192,8 +1518,16 @@ after_credit( fd_admin_tile_ctx_t * ctx, void * payload = NULL; ulong payload_sz = 0UL; + if( FD_UNLIKELY( ctx->failover_slot_idx!=ULONG_MAX && fd_tickcount()>ctx->failover_deadline && + !failov_frag_waiting( ctx, stem ) ) ) { + if( FD_LIKELY( ctx->failover_cmd_cstr[ 0 ] ) ) FD_LOG_WARNING(( "the failover tile did not answer `failover %s` in time, it may still run, check `failover status` and the log", ctx->failover_cmd_cstr )); + else FD_LOG_WARNING(( "the failover tile did not answer `failover status` in time, check the log" )); + failover_complete( ctx, FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE, NULL, 0UL ); + *charge_busy = 1; + } + if( FD_LIKELY( stem->nowdeadline_ticks ) ) return; - ctx->deadline_ticks = stem->now + fd_long_if( ctx->snap_create_slot_idx!=ULONG_MAX, ctx->poll_ticks, ctx->idle_poll_ticks ); + ctx->deadline_ticks = stem->now + fd_long_if( ( ctx->snap_create_slot_idx!=ULONG_MAX || ctx->failover_slot_idx!=ULONG_MAX ), ctx->poll_ticks, ctx->idle_poll_ticks ); ulong cmd_id = FD_ADMINCTL_CMD_IDLE; for( ulong i=0UL; ifailover_enabled ) ) { + failover_request( ctx, stem, slot_idx, payload, payload_sz ); + *charge_busy = 1; + *opt_poll_in = 0; + break; + } + __attribute__((fallthrough)); default: FD_LOG_WARNING(( "unexpected adminctl cmd %lu", cmd_id )); fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ); @@ -1236,6 +1578,14 @@ during_frag( fd_admin_tile_ctx_t * ctx, ulong chunk FD_PARAM_UNUSED, ulong sz FD_PARAM_UNUSED, ulong ctl ) { + if( FD_UNLIKELY( in_idx==ctx->failov_in_idx ) ) { + if( FD_UNLIKELY( chunkfailov_in_chunk0 || chunk>ctx->failov_in_wmark || sz!=sizeof(fd_failover_bus_msg_t) ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->failov_in_chunk0, ctx->failov_in_wmark )); + } + fd_memcpy( &ctx->failov_in, fd_chunk_to_laddr_const( ctx->failov_in_mem, chunk ), sizeof(fd_failover_bus_msg_t) ); + return; + } + if( FD_UNLIKELY( ctx->snap_create_slot_idx==ULONG_MAX ) ) { FD_LOG_ERR(( "unexpected replay snapshot-create response with no pending adminctl command" )); return; @@ -1243,6 +1593,29 @@ during_frag( fd_admin_tile_ctx_t * ctx, snapshot_create_response( ctx, sig, ctl ); } +static void +after_frag( fd_admin_tile_ctx_t * ctx, + ulong in_idx, + ulong seq FD_PARAM_UNUSED, + ulong sig, + ulong sz FD_PARAM_UNUSED, + ulong tsorig FD_PARAM_UNUSED, + ulong tspub FD_PARAM_UNUSED, + fd_stem_context_t * stem ) { + if( FD_LIKELY( in_idx!=ctx->failov_in_idx ) ) return; + switch( sig ) { + case FD_FAILOVER_BUS_SWITCH_REQ: + failover_switch_request( ctx, stem ); + break; + case FD_FAILOVER_BUS_RESPONSE: + failover_response( ctx ); + break; + default: + FD_LOG_WARNING(( "unexpected failover bus frame %lu", sig )); + break; + } +} + static ulong populate_allowed_seccomp( fd_topo_t const * topo FD_PARAM_UNUSED, fd_topo_tile_t const * tile FD_PARAM_UNUSED, @@ -1278,6 +1651,7 @@ populate_allowed_fds( fd_topo_t const * topo FD_PARAM_UNUSED, #define STEM_CALLBACK_NEXT_DEADLINE next_deadline #define STEM_CALLBACK_AFTER_CREDIT after_credit #define STEM_CALLBACK_DURING_FRAG during_frag +#define STEM_CALLBACK_AFTER_FRAG after_frag #include "../../disco/stem/fd_stem.c" diff --git a/src/discof/admin/fd_adminctl.h b/src/discof/admin/fd_adminctl.h index f57c3e24ccc..4f82f3555ab 100644 --- a/src/discof/admin/fd_adminctl.h +++ b/src/discof/admin/fd_adminctl.h @@ -37,6 +37,7 @@ #define FD_ADMINCTL_CMD_GET_IDENTITY (3UL) #define FD_ADMINCTL_CMD_REMOVE_ALL_AUTH_VOTERS (4UL) #define FD_ADMINCTL_CMD_SNAP_CREATE (5UL) +#define FD_ADMINCTL_CMD_FAILOVER (6UL) #define FD_ADMINCTL_ALIGN (8UL) #define FD_ADMINCTL_PAYLOAD_MAX (32768UL) @@ -64,6 +65,20 @@ #define FD_SET_IDENTITY_RESULT_INVALID_VOTE_HISTORY (0x3002UL) #define FD_SET_IDENTITY_RESULT_SAME_IDENTITY (0x3003UL) +#define FD_FAILOVER_CONTROL_RESULT_BUSY (0x4001UL) /* another failover command is waiting on the failover tile */ +#define FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE (0x4002UL) /* the failover tile did not respond in time */ +#define FD_FAILOVER_CONTROL_RESULT_BAD_ROLE (0x4003UL) /* promote on the active, demote on a standby */ +#define FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS (0x4004UL) /* a transition or key switch is running */ +#define FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY (0x4006UL) /* the bound peer cannot complete this handoff */ +#define FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE (0x4007UL) /* the authenticated peer holds the identity */ +#define FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING (0x4008UL) /* the peer has not responded to our handoff */ +#define FD_FAILOVER_CONTROL_RESULT_TAKEN (0x4009UL) /* the peer took our handoff, only a new local tenure clears this */ +#define FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN (0x400AUL) /* gossip has a fresh contact info for the staked identity from another host */ +#define FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER (0x400CUL) /* active has no eligible final vote state, see its local voting diagnostics */ +#define FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS (0x400EUL) /* gossip has no address for the active */ +#define FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED (0x400FUL) /* no check found another holder, which does not prove the peer cannot sign */ +#define FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND (0x4010UL) /* this standby's replay is too far behind the active's last vote */ + struct fd_adminctl_add_auth_voter_v1 { ulong version; /* ==FD_ADMINCTL_ADD_AUTH_VOTER_PAYLOAD_VERSION */ uchar keypair[ 64UL ]; @@ -107,6 +122,121 @@ struct fd_adminctl_remove_all_auth_voters_v1 { typedef struct fd_adminctl_remove_all_auth_voters_v1 fd_adminctl_remove_all_auth_voters_t; #define FD_ADMINCTL_REMOVE_ALL_AUTH_VOTERS_PAYLOAD_VERSION (1UL) +/* Failover commands, forwarded to the failover tile. */ +#define FD_ADMINCTL_FAILOVER_CMD_HANDOFF (0UL) +#define FD_ADMINCTL_FAILOVER_CMD_DEMOTE (1UL) +#define FD_ADMINCTL_FAILOVER_CMD_PROMOTE (2UL) +#define FD_ADMINCTL_FAILOVER_CMD_STATUS (3UL) +#define FD_ADMINCTL_FAILOVER_CMD_CNT (4UL) + +static inline char const * +fd_adminctl_failover_cmd_name( ulong cmd ) { + switch( cmd ) { + case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: return "handoff"; + case FD_ADMINCTL_FAILOVER_CMD_DEMOTE: return "demote"; + case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: return "promote"; + case FD_ADMINCTL_FAILOVER_CMD_STATUS: return "status"; + default: return "unknown"; + } +} + +/* The command as the operator typed it. */ +static inline char const * +fd_adminctl_failover_cmd_typed( ulong cmd ) { + return cmd==FD_ADMINCTL_FAILOVER_CMD_HANDOFF ? "promote" : fd_adminctl_failover_cmd_name( cmd ); +} + +#define FD_ADMINCTL_FAILOVER_FLAG_YES (1UL) /* confirmation only, does not authorize peer or history overrides */ +#define FD_ADMINCTL_FAILOVER_FLAG_FORCE (2UL) /* --force, bypass peer guards and accept incomplete or empty vote history */ + +struct fd_adminctl_failover_req_v1 { + ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ + ulong cmd; /* FD_ADMINCTL_FAILOVER_CMD_* */ + ulong flags; /* FD_ADMINCTL_FAILOVER_FLAG_* */ + uint addr; /* HANDOFF only, the active to dial in network byte order, 0 to find it in gossip */ + ushort port; /* HANDOFF only, the port to dial it on in host byte order, 0 for our own listen port */ + uchar reserved[ 2 ]; +}; +typedef struct fd_adminctl_failover_req_v1 fd_adminctl_failover_req_t; + +struct fd_adminctl_failover_control_resp_v1 { + ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ + uchar role; /* FD_FAILOVER_ROLE_* after the command */ + uchar action; /* controller action after the command */ + uchar reserved[ 6 ]; + ulong handoff_id; /* the handoff an accepted promote or demote opened or resumed, 0 none */ +}; +typedef struct fd_adminctl_failover_control_resp_v1 fd_adminctl_failover_control_resp_t; +#define FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION (1UL) + +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_req_t )==32UL, failover_req_v1_layout ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)==24UL, failover_control_resp_v1_layout ); + +/* What the failover controller is doing right now. */ +#define FD_FAILOVER_ACTION_IDLE (0UL) +#define FD_FAILOVER_ACTION_DEMOTE_SWITCH (1UL) /* waiting for the junk key to be installed */ +#define FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK (2UL) /* DEMOTED sent, waiting for the peer */ +#define FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY (3UL) /* waiting for replay to reach the tower tip */ +#define FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT (4UL) /* waiting for the tower tile to adopt */ +#define FD_FAILOVER_ACTION_PROMOTE_SWITCH (5UL) /* waiting for the staked key to be installed */ +#define FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER (6UL) /* requesting the active's final tower */ +#define FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT (7UL) /* waiting for the old active to record our response */ +#define FD_FAILOVER_ACTION_DEMOTE_DRAIN (8UL) /* junk key installed, waiting for the tower to drain */ +#define FD_FAILOVER_ACTION_CNT (9UL) + +/* The tower sources a promotion can adopt, in order of preference. */ +#define FD_FAILOVER_SOURCE_PEER (0UL) /* the tower the peer's DEMOTED gave us */ +#define FD_FAILOVER_SOURCE_VOTE_ACCOUNT (1UL) /* automatic fallback to the vote account, or forced empty history */ +#define FD_FAILOVER_SOURCE_CNT (2UL) + +/* The outcome of the last handoff this machine requested or gave. */ +#define FD_FAILOVER_HANDOFF_NONE (0UL) +#define FD_FAILOVER_HANDOFF_PENDING (1UL) +#define FD_FAILOVER_HANDOFF_TAKEN (2UL) /* the standby took the identity */ +#define FD_FAILOVER_HANDOFF_DECLINED (3UL) /* the standby did not take the identity, or the old active did not record that it did */ +#define FD_FAILOVER_HANDOFF_RESTARTED (4UL) /* the peer came back with a new boot_id */ +#define FD_FAILOVER_HANDOFF_CANCELLED (5UL) /* `failover promote --force` stopped waiting for the peer */ +#define FD_FAILOVER_HANDOFF_NOT_ACTIVE (6UL) /* the machine we dialed was a standby */ +#define FD_FAILOVER_HANDOFF_CNT (7UL) + +/* `failover status`, provided by the failover tile. */ +struct fd_adminctl_failover_status_resp_v1 { + ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ + uchar enabled; /* 0 when failover is off, the other fields keep their init values then */ + uchar role; /* FD_FAILOVER_ROLE_* */ + uchar action; /* FD_FAILOVER_ACTION_* */ + uchar stuck; /* a transition failed or is overdue */ + uchar link_state; /* FD_FAILOVER_SESSION_* */ + uchar peer_role_valid; /* a peer is authenticated on the current session */ + uchar peer_role; /* FD_FAILOVER_ROLE_* from HELLO or the handoff result */ + uchar handoff_result; /* FD_FAILOVER_HANDOFF_* of our last handoff */ + ulong peer_boot_id; /* boot_id of the last peer we paired with, 0 none */ + uint peer_addr; /* the address we dial, from `failover promote --address` or the active's from gossip, 0 none */ + ushort peer_port; /* the failover port we dial it on, in host byte order */ + uchar promote_source; /* FD_FAILOVER_SOURCE_* `failover promote --force` would adopt now */ + uchar peer_addr_cmd; /* peer_addr is from `failover promote --address`, else it came from gossip */ + ulong handoff_id; /* id of our last handoff, 0 none */ + ulong promote_floor; /* coverage floor a promote has to reach, ULONG_MAX none */ + ulong promote_result; /* what the takeover checks report without --force now, PEER_UNVERIFIED when they find no other holder */ + uchar mode; /* FD_FAILOVER_MODE_* the validator runs */ + uchar request_paused; /* our handoff request stopped at its deadline, `failover promote` resumes it */ + uchar reserved[ 6 ]; +}; +typedef struct fd_adminctl_failover_status_resp_v1 fd_adminctl_failover_status_resp_t; + +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_status_resp_t)==64UL, failover_status_resp_v1_layout ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_status_resp_t)<=FD_ADMINCTL_PAYLOAD_MAX, failover_status_resp_fits ); + +/* fd_adminctl_failover_status_resp_init clears a status response and + sets its version and the unknown promote floor. */ + +static inline void +fd_adminctl_failover_status_resp_init( fd_adminctl_failover_status_resp_t * resp ) { + fd_memset( resp, 0, sizeof(*resp) ); + resp->version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION; + resp->promote_floor = ULONG_MAX; +} + typedef struct fd_adminctl_private fd_adminctl_t; FD_PROTOTYPES_BEGIN diff --git a/src/discof/failover/fd_failover_bus.h b/src/discof/failover/fd_failover_bus.h new file mode 100644 index 00000000000..915b58f413b --- /dev/null +++ b/src/discof/failover/fd_failover_bus.h @@ -0,0 +1,63 @@ +#ifndef HEADER_fd_src_discof_failover_fd_failover_bus_h +#define HEADER_fd_src_discof_failover_fd_failover_bus_h + +/* The admin and failover tiles talk over the admin_failov and + failov_admin links. The admin tile stays the only adminctl poller. It + forwards failover commands with a nonce and parks the adminctl slot + until the response comes back or the deadline passes. The failover + tile asks for identity switches the same way. Every frame is one + fd_failover_bus_msg_t. Each side has at most one request outstanding, + even past the deadline, so neither link has more than two frames in + flight and neither tile can backpressure the other, even behind an + identity switch that never finishes. */ + +#include "../admin/fd_adminctl.h" + +#define FD_FAILOVER_BUS_SWITCH_REQ (1UL) /* failov to admin, fd_failover_switch_req_t */ +#define FD_FAILOVER_BUS_SWITCH_RESP (2UL) /* admin to failov, fd_failover_switch_resp_t */ +#define FD_FAILOVER_BUS_REQUEST (3UL) /* admin to failov, fd_adminctl_failover_req_t */ +#define FD_FAILOVER_BUS_RESPONSE (4UL) /* failov to admin, the requested command's response */ + +/* Only the public key goes over the bus. The sign tile only accepts + a keypair it loaded at boot. */ +struct fd_failover_switch_req { + uchar identity[ 32 ]; /* public key of the identity to install */ +}; + +typedef struct fd_failover_switch_req fd_failover_switch_req_t; + +#define FD_FAILOVER_SWITCH_OK (0UL) +#define FD_FAILOVER_SWITCH_ERR_DISABLED (1UL) /* failover is off here */ + +/* Sent once every tile has switched. */ +struct fd_failover_switch_resp { + ulong result; /* FD_FAILOVER_SWITCH_* */ + ulong tower_watermark; /* the tower keyswitch result, its output sequence at the halt */ +}; + +typedef struct fd_failover_switch_resp fd_failover_switch_resp_t; + +#define FD_FAILOVER_BUS_PAYLOAD_MAX (256UL) + +FD_STATIC_ASSERT( sizeof(fd_failover_switch_req_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, switch_req_fits ); +FD_STATIC_ASSERT( sizeof(fd_failover_switch_resp_t)<=FD_FAILOVER_BUS_PAYLOAD_MAX, switch_resp_fits ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_req_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, command_fits ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)<=FD_FAILOVER_BUS_PAYLOAD_MAX, control_resp_fits ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_status_resp_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, status_resp_fits ); + +struct fd_failover_bus_msg { + ulong nonce; /* per request, echoed on the response */ + ulong result; /* the adminctl result for command responses */ + uchar payload[ FD_FAILOVER_BUS_PAYLOAD_MAX ]; +}; + +typedef struct fd_failover_bus_msg fd_failover_bus_msg_t; + +#define FD_FAILOVER_BUS_MTU (sizeof(fd_failover_bus_msg_t)) +FD_STATIC_ASSERT( sizeof(fd_failover_bus_msg_t)==272UL, bus_layout ); + +/* How long the admin tile waits for a failover command response + before it responds to the operator itself. */ +#define FD_FAILOVER_BUS_DEADLINE_NANOS (2000000000L) + +#endif /* HEADER_fd_src_discof_failover_fd_failover_bus_h */ From d52ac8a710c3111ff0244906184ae385b581e97a Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Thu, 8 Oct 2026 23:02:33 +0000 Subject: [PATCH 09/23] failover: add the on-demand handoff controller --- src/app/firedancer-dev/main.h | 2 + src/app/firedancer/main.c | 2 + src/app/firedancer/topology.c | 69 + src/app/shared/commands/run/run.c | 2 + src/disco/topo/fd_topo.c | 5 + src/disco/topo/fd_topo.h | 11 + src/disco/topo/fd_topo_run.c | 4 +- src/disco/topo/fd_topob_layout.c | 1 + src/discof/failover/Local.mk | 3 + src/discof/failover/fd_failover_tile.c | 2370 +++++++++++++++++ .../failover/fd_failover_tile.seccomppolicy | 42 + .../generated/fd_failover_tile_seccomp.h | 140 + src/discof/failover/test_failover_tile.c | 528 ++++ 13 files changed, 3178 insertions(+), 1 deletion(-) create mode 100644 src/discof/failover/fd_failover_tile.c create mode 100644 src/discof/failover/fd_failover_tile.seccomppolicy create mode 100644 src/discof/failover/generated/fd_failover_tile_seccomp.h create mode 100644 src/discof/failover/test_failover_tile.c diff --git a/src/app/firedancer-dev/main.h b/src/app/firedancer-dev/main.h index cd0bffd2b64..91e109eee07 100644 --- a/src/app/firedancer-dev/main.h +++ b/src/app/firedancer-dev/main.h @@ -117,6 +117,7 @@ extern fd_topo_run_tile_t fd_tile_udpecho; extern fd_topo_run_tile_t fd_tile_genesi; extern fd_topo_run_tile_t fd_tile_ipecho; extern fd_topo_run_tile_t fd_tile_admin; +extern fd_topo_run_tile_t fd_tile_failov; extern fd_topo_run_tile_t fd_tile_gossvf; extern fd_topo_run_tile_t fd_tile_gossip; @@ -185,6 +186,7 @@ fd_topo_run_tile_t * TILES[] = { &fd_tile_genesi, &fd_tile_ipecho, &fd_tile_admin, + &fd_tile_failov, &fd_tile_solcap, &fd_tile_snapmk, &fd_tile_snapzp, diff --git a/src/app/firedancer/main.c b/src/app/firedancer/main.c index d99ad4482b5..adae39a2259 100644 --- a/src/app/firedancer/main.c +++ b/src/app/firedancer/main.c @@ -106,6 +106,7 @@ extern fd_topo_run_tile_t fd_tile_bundle; extern fd_topo_run_tile_t fd_tile_genesi; extern fd_topo_run_tile_t fd_tile_ipecho; extern fd_topo_run_tile_t fd_tile_admin; +extern fd_topo_run_tile_t fd_tile_failov; extern fd_topo_run_tile_t fd_tile_gossvf; extern fd_topo_run_tile_t fd_tile_gossip; @@ -163,6 +164,7 @@ fd_topo_run_tile_t * TILES[] = { &fd_tile_genesi, &fd_tile_ipecho, &fd_tile_admin, + &fd_tile_failov, &fd_tile_solcap, &fd_tile_snapmk, &fd_tile_snapzp, diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index 768ca1107c0..e5fbacc1fea 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -11,6 +11,8 @@ #include "../../discof/votor/fd_votor_tile.h" #include "../../disco/keyguard/fd_keyguard.h" #include "../../ballet/bls/fd_bls.h" +#include "../../discof/failover/fd_failover_bus.h" +#include "../../discof/failover/fd_failover_proto.h" #include "../../discof/backup/fd_snapmk_tile.h" #include "../../discof/backup/fd_snapsv_tile.h" #include "../../disco/shred/fd_shred_tile.h" @@ -283,6 +285,7 @@ fd_topo_initialize( config_t * config ) { int leader_enabled = !!config->firedancer.layout.enable_block_production; int rserve_enabled = config->tiles.rserve.enabled; int efficient_mode = !strcmp( config->firedancer.layout.mode, "efficient" ); + int failover_enabled = config->firedancer.failover.enabled; if( FD_UNLIKELY( snapmk_enabled ) ) { FD_CHECK_ERR( config->firedancer.snapshots.max_full_snapshots_to_keep, @@ -315,6 +318,9 @@ fd_topo_initialize( config_t * config ) { fd_topob_wksp( topo, "votor" ); fd_topob_wksp( topo, "sign" )->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_NEVER; fd_topob_wksp( topo, "admin" )->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_NEVER; + /* The failov tile keeps TLS session keys in its scratch, so its + workspace stays out of core dumps like admin and sign. */ + if( FD_UNLIKELY( failover_enabled ) ) fd_topob_wksp( topo, "failov" )->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_NEVER; if( leader_enabled ) { fd_topob_wksp( topo, "quic" ); @@ -402,8 +408,20 @@ fd_topo_initialize( config_t * config ) { fd_topob_wksp( topo, "votor_sign" ); fd_topob_wksp( topo, "sign_votor" ); + if( FD_UNLIKELY( failover_enabled ) ) { + fd_topob_wksp( topo, "failov_sign" ); + fd_topob_wksp( topo, "sign_failov" ); + } + + fd_topob_wksp( topo, "execrp_replay" ); fd_topob_wksp( topo, "admin_replay" ); + if( FD_UNLIKELY( failover_enabled ) ) { + fd_topob_wksp( topo, "admin_failov" ); + fd_topob_wksp( topo, "failov_admin" ); + if( !alpenglow_enabled ) fd_topob_wksp( topo, "adopt_tower" ); + if( !config->tiles.gui.enabled ) fd_topob_wksp( topo, "gossip_ciseen" ); + } if( FD_LIKELY( snapshots_enabled ) ) { fd_topob_wksp( topo, "snapct" ); @@ -506,6 +524,17 @@ fd_topo_initialize( config_t * config ) { FOR(execrp_tile_cnt) fd_topob_link( topo, "replay_execrp", "replay_execrp", 256UL, sizeof(fd_execrp_task_msg_t), 1UL ); /**/ fd_topob_link( topo, "admin_replay", "admin_replay", 32UL, 0UL, 1UL ); /**/ fd_topob_link( topo, "replay_admin", "admin_replay", 32UL, 0UL, 1UL ); + if( FD_UNLIKELY( failover_enabled ) ) { + /**/ fd_topob_link( topo, "admin_failov", "admin_failov", 32UL, FD_FAILOVER_BUS_MTU, 1UL ); + /**/ fd_topob_link( topo, "failov_admin", "failov_admin", 32UL, FD_FAILOVER_BUS_MTU, 1UL ); + if( !alpenglow_enabled ) { + /**/ fd_topob_link( topo, "adopt_tower", "adopt_tower", 32UL, FD_FAILOVER_TOWER_STATE_MAX, 1UL ); + /**/ fd_topob_link( topo, "tower_adopt", "adopt_tower", 32UL, sizeof(fd_tower_adopt_result_t), 1UL ); + } + if( !config->tiles.gui.enabled ) { + fd_topob_link( topo, "gossip_ciseen", "gossip_ciseen", 65536UL*2UL, sizeof(fd_gossip_update_message_t), 1UL ); + } + } if( leader_enabled ) { /**/ fd_topob_link( topo, "dedup_resolv", "dedup_resolv", 16384UL, FD_TPU_PARSED_MTU, 1UL ); FOR(resolv_tile_cnt) fd_topob_link( topo, "resolv_pack", "resolv_pack", 4096UL, FD_TPU_RESOLVED_MTU, 1UL ); @@ -530,6 +559,11 @@ fd_topo_initialize( config_t * config ) { FOR(sign_tile_cnt-1) fd_topob_link( topo, "repair_sign", "repair_sign", 256UL, FD_REPAIR_SIG_SER_MAX, 1UL ); /* See repair_tile.c for explanation */ FOR(sign_tile_cnt-1) fd_topob_link( topo, "sign_repair", "sign_repair", 256UL, sizeof(fd_ed25519_sig_t), 1UL ); + if( FD_UNLIKELY( failover_enabled ) ) { + /**/ fd_topob_link( topo, "failov_sign", "failov_sign", 128UL, 130UL, 1UL ); /* member certificate or TLS 1.3 CertificateVerify payload */ + /**/ fd_topob_link( topo, "sign_failov", "sign_failov", 128UL, sizeof(fd_ed25519_sig_t), 1UL ); + } + FOR(shred_tile_cnt) fd_topob_link( topo, "shred_out", "shred_out", shred_depth, sizeof(fd_shred_message_t), FD_SHRED_STEM_BURST ); /**/ fd_topob_link( topo, "repair_out", "repair_out", repair_out_depth, sizeof(fd_repair_replay_fec_t), 1UL ); @@ -633,6 +667,7 @@ fd_topo_initialize( config_t * config ) { } fd_topo_tile_t * admin_tile = fd_topob_tile( topo, "admin", "admin", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 ); + if( FD_UNLIKELY( failover_enabled ) ) fd_topob_tile( topo, "failov", "failov", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 ); /* topo, tile_name, tile_kind_id, fseq_wksp, link_name, link_kind_id, reliable, polled */ FOR(gossvf_tile_cnt) for( ulong j=0UL; jtiles.gui.enabled ) { + fd_topob_tile_out( topo, "gossip", 0UL, "gossip_ciseen", 0UL ); + } + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "gossip_ciseen", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* contact infos for the peer's address, an overrun only delays it */ + /**/ fd_topob_tile_out( topo, "admin", 0UL, "admin_failov", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "admin_failov", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_admin", 0UL ); + /**/ fd_topob_tile_in ( topo, "admin", 0UL, "metric_in", "failov_admin", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + } + if( !alpenglow_enabled && failover_enabled ) { + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "adopt_tower", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_adopt", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + } FOR(execrp_tile_cnt) fd_topob_tile_in ( topo, "execrp", i, "metric_in", "replay_execrp", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); FOR(execrp_tile_cnt) fd_topob_tile_out( topo, "execrp", i, "execrp_replay", i ); @@ -892,6 +942,13 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_rserve", 0UL ); } + if( FD_UNLIKELY( failover_enabled ) ) { + /**/ fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "failov_sign", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_sign", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "sign_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED ); + /**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_failov", 0UL ); + } + if( FD_UNLIKELY( rpc_enabled ) ) { fd_topob_link( topo, "rpc_replay", "rpc_replay", 8UL, 0UL, 1UL ); fd_topob_tile_out( topo, "rpc", 0UL, "rpc_replay", 0UL ); @@ -1449,6 +1506,18 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, fd_cstr_ncpy( tile->admin.identity_key_path, identity_key_path, sizeof(tile->admin.identity_key_path) ); + } else if( FD_UNLIKELY( !strcmp( tile->name, "failov" ) ) ) { + + fd_cstr_ncpy( tile->failov.identity_key_path, identity_key_path, sizeof(tile->failov.identity_key_path) ); + fd_cstr_ncpy( tile->failov.staked_key_path, config->paths.identity_key, sizeof(tile->failov.staked_key_path) ); + fd_cstr_ncpy( tile->failov.vote_account_path, config->paths.vote_account, sizeof(tile->failov.vote_account_path) ); + if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->firedancer.failover.listen_address, &tile->failov.listen_addr ) ) ) + FD_LOG_ERR(( "[failover.listen_address] `%s` must be an IPv4 address, host names are not accepted", config->firedancer.failover.listen_address )); + tile->failov.port = config->firedancer.failover.listen_port; + fd_cstr_ncpy( tile->failov.gossip_host, config->firedancer.gossip.host, sizeof(tile->failov.gossip_host) ); + tile->failov.gossip_addr.addr = config->net.ip_addr; + tile->failov.gossip_addr.port = fd_ushort_bswap( config->gossip.port ); + } else if( FD_UNLIKELY( !strcmp( tile->name, "gossvf") ) ) { fd_cstr_ncpy( tile->gossvf.identity_key_path, identity_key_path, sizeof(tile->gossvf.identity_key_path) ); diff --git a/src/app/shared/commands/run/run.c b/src/app/shared/commands/run/run.c index 8f9af899632..139c4d02739 100644 --- a/src/app/shared/commands/run/run.c +++ b/src/app/shared/commands/run/run.c @@ -82,6 +82,8 @@ run_cmd_perm( args_t * args, fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for serving metrics" ); if( FD_UNLIKELY( config->tiles.gui.gui_listen_port<1024 ) ) fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for serving the GUI" ); + if( FD_UNLIKELY( config->is_firedancer && config->firedancer.failover.enabled && config->firedancer.failover.listen_port<1024 ) ) + fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for the failover listener" ); } struct pidns_clone_args { diff --git a/src/disco/topo/fd_topo.c b/src/disco/topo/fd_topo.c index 7ae13915287..55132fb5865 100644 --- a/src/disco/topo/fd_topo.c +++ b/src/disco/topo/fd_topo.c @@ -237,6 +237,11 @@ fd_topo_tile_extra_normal_pages( fd_topo_tile_t const * tile ) { key material. */ key_pages += 5UL; } + if( FD_UNLIKELY( !strcmp( tile->name, "failov" ) ) ) { + /* The failover tile loads the junk and staked public keys at boot, + both at the same time. */ + key_pages += 10UL; + } if( !strcmp( tile->name, "net" ) ) { /* net tile uses normal pages to hold xsk rings */ diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index 6fb6cd1a318..22ebd907c0b 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -245,6 +245,17 @@ struct fd_topo_tile { char identity_key_path[ PATH_MAX ]; } admin; + struct { + char identity_key_path[ PATH_MAX ]; /* this machine's junk key */ + char staked_key_path[ PATH_MAX ]; /* we read only its public half */ + char vote_account_path[ PATH_MAX ]; + uint listen_addr; /* network byte order */ + ushort port; + + char gossip_host[ FD_FQDN_BUF_MAX ]; + fd_ip4_port_t gossip_addr; /* our own gossip socket, gossip_host overrides the address */ + } failov; + #define FD_TOPO_GOSSIP_ENTRYPOINTS_MAX 16UL struct { diff --git a/src/disco/topo/fd_topo_run.c b/src/disco/topo/fd_topo_run.c index 38ee1e711af..08fbf80c7cd 100644 --- a/src/disco/topo/fd_topo_run.c +++ b/src/disco/topo/fd_topo_run.c @@ -232,7 +232,9 @@ fd_topo_tile_stack_join( char const * app_name, char name[ PATH_MAX ]; FD_TEST( fd_cstr_printf_check( name, PATH_MAX, NULL, "%s_stack_%s%lu", app_name, tile_name, tile_kind_id ) ); - int dump = strcmp( tile_name, "sign" ) ? 1 : 0; /* avoid core dumps of sign tile stacks */ + /* avoid core dumps of sign tile stacks, and of failov ones, which hold + TLS session secrets */ + int dump = strcmp( tile_name, "sign" ) && strcmp( tile_name, "failov" ) ? 1 : 0; uchar * stack = fd_shmem_join( name, FD_SHMEM_JOIN_MODE_READ_WRITE, dump, NULL, NULL, NULL ); if( FD_UNLIKELY( !stack ) ) FD_LOG_ERR(( "fd_shmem_join failed" )); diff --git a/src/disco/topo/fd_topob_layout.c b/src/disco/topo/fd_topob_layout.c index a80d6251ee0..80d727b72b1 100644 --- a/src/disco/topo/fd_topob_layout.c +++ b/src/disco/topo/fd_topob_layout.c @@ -72,6 +72,7 @@ static struct { char const * name; uint flags; } const TILE_CLASS[] = { { "genesi", FLOATING }, { "ipecho", FLOATING }, { "admin", FLOATING }, + { "failov", FLOATING }, { "snapsv", FLOATING|PIN_EFF }, { "snapmk", FLOATING }, { "snapzp", FLOATING }, diff --git a/src/discof/failover/Local.mk b/src/discof/failover/Local.mk index afacd4d010b..d748b75a66e 100644 --- a/src/discof/failover/Local.mk +++ b/src/discof/failover/Local.mk @@ -6,10 +6,13 @@ ifdef FD_HAS_HOSTED $(call add-hdrs,fd_failover_channel.h) $(call add-objs,fd_failover_channel,fd_discof) $(call add-objs,fd_failover_tls,fd_discof) +$(call add-objs,fd_failover_tile,fd_discof) $(call make-unit-test,test_failover_proto,test_failover_proto,fd_discof fd_choreo fd_flamenco fd_tango fd_ballet fd_util) $(call run-unit-test,test_failover_proto) $(call make-unit-test,test_failover_wire,test_failover_wire,fd_discof fd_ballet fd_util) $(call run-unit-test,test_failover_wire) $(call make-unit-test,test_failover_channel,test_failover_channel,fd_discof fd_disco fd_choreo fd_flamenco fd_waltz fd_tls fd_tango fd_ballet fd_util) $(call run-unit-test,test_failover_channel) +$(call make-unit-test,test_failover_tile,test_failover_tile,fd_discof fd_disco fd_choreo fd_flamenco fd_waltz fd_tls fd_tango fd_ballet fd_util) +$(call run-unit-test,test_failover_tile) endif diff --git a/src/discof/failover/fd_failover_tile.c b/src/discof/failover/fd_failover_tile.c new file mode 100644 index 00000000000..3d271628453 --- /dev/null +++ b/src/discof/failover/fd_failover_tile.c @@ -0,0 +1,2370 @@ +#include "../../disco/topo/fd_topo.h" +#include "../../disco/topo/fd_dns_resolve.h" +#include "../../disco/keyguard/fd_keyload.h" +#include "../../disco/keyguard/fd_keyguard.h" +#include "../../disco/keyguard/fd_keyguard_client.h" +#include "../../flamenco/gossip/fd_gossip_message.h" +#include "../../ballet/base58/fd_base58.h" +#include "../../ballet/hex/fd_hex.h" +#include "../../ballet/txn/fd_txn.h" +#include "../../choreo/tower/fd_tower_serdes.h" +#include "../../util/fd_version.h" +#include "../../util/net/fd_ip4.h" +#include "../tower/fd_tower_tile.h" + +#include "fd_failover_bus.h" +#include "fd_failover_channel.h" + +#include +#include + +#include "generated/fd_failover_tile_seccomp.h" + +/* The failov tile owns the socket to the other failover machine and is + the only tile that keeps host networking for it. It holds no private + key. The sign tile signs our member certificate with the staked key + and our TLS handshakes with the junk key. The admin tile keeps the + identity switch and stays off the network. */ + +/* Idle socket poll spacing */ +#define FD_FAILOVER_TILE_POLL_NANOS (50000L) /* 50us */ + +/* A transition gives up once replay passes this many slots past its + start. */ +#define FD_FAILOVER_DEADLINE_SLOTS (64UL) + +/* A vote account tower short of the coverage floor is taken once replay + passes the floor by this much. */ +#define FD_FAILOVER_FLOOR_SLACK_SLOTS (512UL) + +/* The promotion's replay and adopt waits also end on our clock, one + second per deadline slot, so a standby whose replay froze stops. */ +#define FD_FAILOVER_DEADLINE_SLOT_NANOS (1000000000L) + +/* A staked contact info from another host newer than this means an + active is publishing, two gossip refreshes of 7.5s. */ +#define FD_FAILOVER_GOSSIP_FRESH_NANOS (15000000000L) + +/* The controller actions, promotion sources and handoff results are + in fd_adminctl.h, `failover status` reports them. */ + +/* Which key a switch installs, CNT while none is in flight */ +#define FD_FAILOVER_SWITCH_KEY_JUNK (0UL) +#define FD_FAILOVER_SWITCH_KEY_STAKED (1UL) +#define FD_FAILOVER_SWITCH_KEY_CNT (2UL) + +/* Our DEMOTED to the handoff target */ +#define FD_FAILOVER_DELIVERY_NONE (0UL) /* Nothing to send. */ +#define FD_FAILOVER_DELIVERY_PENDING (1UL) /* Not sent on this session yet. */ +#define FD_FAILOVER_DELIVERY_SENT (2UL) /* Sent on this session, a new session sends it again. */ + +/* Our last response to a DEMOTED */ +#define FD_FAILOVER_REPLY_NONE (0UL) /* No response yet. */ +#define FD_FAILOVER_REPLY_KEPT (1UL) /* Kept, the same DEMOTED again gets it again. */ +#define FD_FAILOVER_REPLY_PENDING (2UL) /* Kept, not queued yet because the control slot was busy. */ + +/* A compact tower and the slot of its last vote. */ +struct fd_failover_tower { + int valid; /* holds a tower */ + ulong tip; /* slot of its last vote */ + ulong sz; /* bytes used in state */ + uchar state[ FD_FAILOVER_TOWER_STATE_MAX ]; /* the serialized tower */ +}; + +typedef struct fd_failover_tower fd_failover_tower_t; + +/* The member and boot a message or request is bound to. */ +struct fd_failover_peer { + ulong boot_id; /* the peer's boot */ + uchar junk[ 32 ]; /* the peer's junk key */ +}; + +typedef struct fd_failover_peer fd_failover_peer_t; + +static inline int +peer_eq( fd_failover_peer_t const * bound, + ulong boot_id, + uchar const * junk ) { + return bound->boot_id==boot_id && fd_memeq( bound->junk, junk, 32UL ); +} + +/* The session we last saw on the channel. */ +struct fd_failover_session { + ulong state; /* last seen session state, for edge detection */ + ulong generation; /* catches a replacement paired during one poll */ + ulong peer_boot_id; /* boot of the peer on this session */ + ulong peer_role; /* latest HELLO or accepted handoff seen on this session */ + int close_after_send; /* hang up once the queued message is sent */ + ulong close_handoff_id; /* the handoff that hang up is for, for the log */ +}; + +typedef struct fd_failover_session fd_failover_session_t; + +/* Our handoff request, sent by the standby. */ +struct fd_failover_request { + ulong id; /* zero when none is open */ + fd_failover_peer_t peer; /* the active boot we authenticated for it */ + uint addr; /* the address we dial, a resume may replace it */ + long until; /* when the request pauses, zero while paused */ + int sent; /* sent on this session */ + int peer_restarted; /* paused because the boot we authenticated restarted */ + ulong last_id; /* id of our latest request, kept for status */ + ulong result; /* FD_FAILOVER_HANDOFF_* of our latest request */ + ulong send_cnt; /* how many times we sent it */ + long log_at; /* when we may log a resend again */ +}; + +typedef struct fd_failover_request fd_failover_request_t; + +/* Our demotion. */ +struct fd_failover_handoff { + ulong id; /* id of our latest demotion */ + fd_failover_peer_t target; /* peer boot it is for, boot_id 0 for a demote */ + ulong result; /* FD_FAILOVER_HANDOFF_* of the latest DEMOTED */ + ulong delivery; /* FD_FAILOVER_DELIVERY_* */ + long until; /* the demotion step's deadline on our clock, for when replay stops */ + long log_at; /* when we may log a resend again */ + long result_log_at; /* when we may log the result again */ + ulong payload_sz; /* bytes used in payload */ + uchar payload[ FD_FAILOVER_DEMOTED_PAYLOAD_MAX ]; /* our DEMOTED, resent on a new session */ +}; + +typedef struct fd_failover_handoff fd_failover_handoff_t; + +/* The promotion in progress. */ +struct fd_failover_promotion { + char label[ 48 ]; /* handoff ID or operator promotion, fixed for every step */ + ulong source; /* FD_FAILOVER_SOURCE_* we adopt from */ + int empty; /* forced recovery after the vote account could not be adopted */ + int from_peer; /* the peer's DEMOTED requested it, we send the response */ + fd_failover_peer_t peer; /* that DEMOTED's member boot, it gets our response */ + ulong handoff_id; /* and its handoff id */ + ulong floor; /* coverage floor when it started */ + int force; /* `failover promote --force`, we skip the peer checks and accept incomplete or empty history */ + int active_seen; /* an ACTIVE peer authenticated after promotion started */ + long until; /* our clock when the replay and adopt waits give up, 0 until set */ + fd_failover_tower_t adopt; /* what goes to the tower tile, empty for the vote account */ + ulong adopt_id; /* id of the adoption we asked the tower tile for */ + int retry; /* ask the tower tile again once replay passes retry_slot */ + ulong retry_slot; /* replay slot that retry waits for */ + int floor_logged; /* we said once that we wait for the floor */ + int retry_logged; /* and that we ask the tower tile again as replay moves */ +}; + +typedef struct fd_failover_promotion fd_failover_promotion_t; + +/* Our last response to a DEMOTED. The same DEMOTED again gets the same + response, so a peer that missed it is not left waiting. */ +struct fd_failover_reply { + ulong state; /* FD_FAILOVER_REPLY_* */ + fd_failover_peer_t peer; /* the member boot it goes to */ + ulong handoff_id; /* the handoff it answers */ + ushort type; /* PROMOTE_ACK or PROMOTE_REJECTED */ + uchar reason; /* for a rejected reply */ + long log_at; /* when we may log a resend again */ +}; + +typedef struct fd_failover_reply fd_failover_reply_t; + +/* The one control message waiting to go out, if any. */ +struct fd_failover_tx { + ushort type; /* FD_FAILOVER_MSG_* */ + ushort sz; /* bytes used in payload */ + int valid; /* a message is waiting */ + fd_failover_peer_t to; /* the member boot it goes to */ + uchar payload[ FD_FAILOVER_DEMOTED_PAYLOAD_MAX ]; /* the encoded message */ +}; + +typedef struct fd_failover_tx fd_failover_tx_t; + +/* State of the identity switch we asked the admin tile for. It + selects a key preloaded by the signing tile by its public key, and + success means the old key is gone from every tile's active identity. */ +struct fd_failover_id_switch { + ulong request_id; /* nonce of our latest switch request */ + ulong pending_key; /* FD_FAILOVER_SWITCH_KEY_*, or CNT when idle */ + fd_failover_switch_resp_t result; /* the committed response */ + fd_failover_switch_resp_t response; /* uncommitted input frame */ + ulong response_nonce; /* nonce of the frame being consumed */ + int fresh; /* a response frame arrived and is not handled yet */ + int overdue; /* the switch in flight passed its deadline, we still wait for it */ +}; + +typedef struct fd_failover_id_switch fd_failover_id_switch_t; + +struct fd_failover_tile_ctx { + fd_keyguard_client_t keyguard_client[ 1 ]; /* signs the member certificate */ + int member_cert_set; /* hello carries a signed member certificate */ + + fd_failover_hello_t hello; /* the HELLO we send */ + ulong role; /* FD_FAILOVER_ROLE_* */ + uint cmd_addr; /* from `failover promote --address`, 0 when gossip finds the active */ + ushort cmd_port; /* from `failover promote --port`, 0 for our own port */ + ushort port; /* our failover listen port */ + fd_failover_channel_t * channel; /* the TLS channel to the peer */ + fd_failover_session_t session; /* the session we last saw on the channel */ + long poll_at; /* next idle socket poll */ + + fd_failover_request_t request; /* our handoff request, as a standby */ + int last_requested; /* status describes our request rather than our demotion */ + long handoff_refuse_log_at; /* when we may log a refused handoff request again */ + + ulong peer_floor; /* highest last vote the peer reported, kept across sessions */ + ulong own_floor; /* highest vote we signed while ACTIVE this boot */ + + /* gossip_ciseen, polled unreliably for contact infos */ + ulong gossip_in_idx; /* gossip_ciseen in link */ + fd_wksp_t * gossip_in_mem; /* its workspace */ + ulong gossip_in_chunk0; /* its first chunk */ + ulong gossip_in_wmark; /* its last chunk */ + ulong gossip_in_mtu; /* its largest frag */ + + fd_ip4_port_t own_gossip; /* our own gossip socket */ + uchar gossip_origin[ 32 ]; /* copied in during_frag */ + fd_ip4_port_t gossip_socket; /* copied in during_frag, zero for a remove or IPv6 */ + uint staked_addr; /* from a staked contact info that is not ours, 0 none */ + long staked_seen_at; /* our clock when that one arrived, 0 never */ + + /* Controller */ + ulong action; /* FD_FAILOVER_ACTION_* */ + int stuck; /* a transition failed or is overdue, shown to the operator */ + int stuck_logged; /* the stuck value we last told the operator about */ + ulong deadline_slot; /* replay slot past which the current step is overdue */ + + /* Handoff ids count up from a random base per boot, so ids from + different boots of ours do not collide. */ + ulong handoff_base; /* random base for our handoff ids */ + ulong handoff_cnt; /* handoff ids issued since boot */ + fd_failover_handoff_t handoff; /* our demotion */ + int taken; /* the peer took the identity, only a new tenure clears this */ + + fd_failover_tower_t peer_tower; /* stored peer tower, from a DEMOTED */ + + fd_failover_promotion_t promotion; /* the promotion in progress */ + fd_failover_reply_t reply; /* our last response to a DEMOTED */ + fd_failover_tx_t tx; /* the control message waiting to go out */ + + /* The admin tile forwards operator commands over the bus and waits for + one response frame per request. */ + ulong admin_in_idx; /* admin_failov in link */ + fd_wksp_t * admin_in_mem; /* its workspace */ + ulong admin_in_chunk0; /* its first chunk */ + ulong admin_in_wmark; /* its last chunk */ + ulong admin_out_idx; /* failov_admin out link */ + fd_wksp_t * admin_out_mem; /* its workspace */ + ulong admin_out_chunk0; /* its first chunk */ + ulong admin_out_wmark; /* its last chunk */ + ulong admin_out_chunk; /* next chunk we publish to */ + fd_failover_bus_msg_t bus_req; /* the operator command copied in during_frag */ + int bus_req_fresh; /* bus_req arrived and is not handled yet */ + + fd_failover_id_switch_t id_switch; /* the identity switch we asked the admin tile for */ + + /* Adoption request to the tower tile and its response. */ + ulong adopt_out_idx; /* adoption request out link */ + fd_wksp_t * adopt_out_mem; /* its workspace */ + ulong adopt_out_chunk0; /* its first chunk */ + ulong adopt_out_wmark; /* its last chunk */ + ulong adopt_out_chunk; /* next chunk we publish to */ + ulong adopt_request_id; /* id of our latest adoption request */ + ulong adopt_in_idx; /* adoption response in link */ + fd_wksp_t * adopt_in_mem; /* its workspace */ + ulong adopt_in_chunk0; /* its first chunk */ + ulong adopt_in_wmark; /* its last chunk */ + fd_tower_adopt_result_t adopt_result; /* the adoption response copied in during_frag */ + ulong adopt_result_id; /* the request it answers */ + int adopt_result_fresh; /* adopt_result arrived and is not handled yet */ + + /* tower_out, polled unreliably */ + ulong tower_in_idx; /* tower_out in link */ + fd_wksp_t * tower_in_mem; /* its workspace */ + ulong tower_in_chunk0; /* its first chunk */ + ulong tower_in_wmark; /* its last chunk */ + + fd_tower_slot_done_t slot_done; /* the tower_out frame copied in during_frag */ + int slot_done_fresh; /* slot_done arrived and is not handled yet */ + ulong tower_seen_seq; /* seq of the last tower_out frag taken in, ULONG_MAX before any */ + int tower_gap; /* a tower_out frag was skipped since the cached tower was built */ + + ulong replay_slot; /* latest replayed slot */ + ulong root_slot; /* latest root */ + ulong last_vote_slot; /* our latest vote */ + + fd_failover_tower_t current_tower; /* tower of our latest vote */ + + uchar rx[ FD_FAILOVER_PAYLOAD_MAX ]; /* the control message being decoded */ +}; + +typedef struct fd_failover_tile_ctx fd_failover_tile_ctx_t; + +FD_FN_CONST static inline ulong +scratch_align( void ) { + return fd_ulong_max( alignof(fd_failover_tile_ctx_t), fd_failover_channel_align() ); +} + +FD_FN_PURE static inline ulong +scratch_footprint( fd_topo_tile_t const * tile FD_PARAM_UNUSED ) { + ulong l = FD_LAYOUT_INIT; + l = FD_LAYOUT_APPEND( l, alignof(fd_failover_tile_ctx_t), sizeof(fd_failover_tile_ctx_t) ); + l = FD_LAYOUT_APPEND( l, fd_failover_channel_align(), fd_failover_channel_footprint() ); + return FD_LAYOUT_FINI( l, scratch_align() ); +} + +/* TLS CertificateVerify, signed by the sign tile with the junk key. */ +static void +sign_ed25519( void * signer_ctx, + uchar sig[ static FD_ED25519_SIG_SZ ], + uchar const msg[ static FD_TLS_CV_SIGN_SZ ] ) { + fd_failover_tile_ctx_t * ctx = signer_ctx; + fd_keyguard_client_sign( ctx->keyguard_client, sig, msg, FD_TLS_CV_SIGN_SZ, FD_KEYGUARD_SIGN_TYPE_ED25519 ); +} + +static void +privileged_init( fd_topo_t const * topo, + fd_topo_tile_t const * tile ) { + void * scratch = fd_topo_obj_laddr( topo, tile->tile_obj_id ); + FD_SCRATCH_ALLOC_INIT( l, scratch ); + fd_failover_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_failover_tile_ctx_t), sizeof(fd_failover_tile_ctx_t) ); + void * channel_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_failover_channel_align(), fd_failover_channel_footprint() ); + fd_memset( ctx, 0, sizeof(fd_failover_tile_ctx_t) ); + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT; + ctx->deadline_slot = FD_FAILOVER_SLOT_NULL; + ctx->peer_floor = FD_FAILOVER_SLOT_NULL; + ctx->own_floor = FD_FAILOVER_SLOT_NULL; + ctx->replay_slot = FD_FAILOVER_SLOT_NULL; + ctx->root_slot = FD_FAILOVER_SLOT_NULL; + ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; + ctx->tower_seen_seq = ULONG_MAX; + FD_TEST( fd_rng_secure( &ctx->handoff_base, sizeof(ulong) ) ); + + /* The junk identity is this machine's own key, it goes into HELLO and + TLS. We keep only the public keys of it and of the staked + [paths.identity_key]. Like every tile's identity load, the loader + reads the whole file and zeroes the private half before it returns. */ + uchar const * junk_pubkey = fd_keyload_load( tile->failov.identity_key_path, 1 ); + uchar const * staked_pubkey = fd_keyload_load( tile->failov.staked_key_path, 1 ); /* public key only, like the other tiles' identity loads */ + fd_memcpy( ctx->hello.junk_pubkey, junk_pubkey, 32UL ); + fd_memcpy( ctx->hello.staked_pubkey, staked_pubkey, 32UL ); + fd_keyload_unload( junk_pubkey, 1 ); + fd_keyload_unload( staked_pubkey, 1 ); + + if( FD_UNLIKELY( fd_memeq( ctx->hello.junk_pubkey, ctx->hello.staked_pubkey, 32UL ) ) ) { + FD_LOG_ERR(( "the failover junk identity `%s` is the staked [paths.identity_key], each failover machine boots under its own key", tile->failov.identity_key_path )); + } + + /* The vote key is a base58 pubkey or a keypair file, same as in the + tower tile. */ + if( FD_UNLIKELY( !fd_base58_decode_32( tile->failov.vote_account_path, ctx->hello.vote_account ) ) ) { + if( FD_UNLIKELY( !strcmp( tile->failov.vote_account_path, "" ) ) ) FD_LOG_ERR(( "missing [paths.vote_account]" )); + uchar const * vote_account = fd_keyload_load( tile->failov.vote_account_path, 1 ); + fd_memcpy( ctx->hello.vote_account, vote_account, 32UL ); + fd_keyload_unload( vote_account, 1 ); + } + + /* Every boot starts passively, as a standby on the junk key. */ + ctx->role = FD_FAILOVER_ROLE_STANDBY; + ctx->hello.version = (ushort)FD_FAILOVER_VERSION; + ctx->hello.role = (uchar)ctx->role; + ctx->hello.mode = (uchar)FD_FAILOVER_MODE_TOWER; + while( FD_UNLIKELY( !ctx->hello.boot_id ) ) FD_TEST( fd_rng_secure( &ctx->hello.boot_id, sizeof(ulong) ) ); + /* The commit field has the hash bytes, a build without one leaves it + zero. */ + if( FD_LIKELY( strlen( fd_commit_ref_cstr )==2UL*sizeof(ctx->hello.commit) && + fd_hex_decode( ctx->hello.commit, fd_commit_ref_cstr, sizeof(ctx->hello.commit) )!=sizeof(ctx->hello.commit) ) ) { + fd_memset( ctx->hello.commit, 0, sizeof(ctx->hello.commit) ); + } + ctx->port = tile->failov.port; + + ctx->own_gossip = tile->failov.gossip_addr; + if( FD_UNLIKELY( tile->failov.gossip_host[ 0 ]!='\0' ) ) { + if( FD_UNLIKELY( !fd_dns_resolve_address( tile->failov.gossip_host, &ctx->own_gossip.addr ) ) ) { + FD_LOG_ERR(( "could not resolve [gossip.host] `%s`", tile->failov.gossip_host )); + } + } + + ctx->channel = fd_failover_channel_join( fd_failover_channel_new( channel_mem ) ); + FD_TEST( ctx->channel ); + fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 }; + if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, &ctx->hello ) ) ) { + FD_LOG_ERR(( "failover TLS setup failed for the junk identity `%s`", tile->failov.identity_key_path )); + } + + fd_failover_channel_init_listener( ctx->channel, tile->failov.listen_addr, tile->failov.port ); + ctx->session.state = fd_failover_channel_state( ctx->channel ); + + char junk_b58 [ FD_BASE58_ENCODED_32_SZ ]; + char staked_b58[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( ctx->hello.junk_pubkey, NULL, junk_b58 ); + fd_base58_encode_32( ctx->hello.staked_pubkey, NULL, staked_b58 ); + FD_LOG_NOTICE(( "failover is on, booting as a standby under junk identity `%s` for staked identity `%s`, listening on `" FD_IP4_ADDR_FMT ":%u`, " + "this machine does not vote until `failover promote` hands it the identity from the active, or `failover promote --force` takes it when no machine holds it, " + "the junk identity is this machine's own unique key, never copy it", junk_b58, staked_b58, FD_IP4_ADDR_FMT_ARGS( tile->failov.listen_addr ), (uint)tile->failov.port )); + + FD_TEST( FD_SCRATCH_ALLOC_FINI( l, scratch_align() )==(ulong)scratch+scratch_footprint( tile ) ); +} + +static void +unprivileged_init( fd_topo_t const * topo, + fd_topo_tile_t const * tile ) { + fd_failover_tile_ctx_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id ); + + ctx->gossip_in_idx = ULONG_MAX; + ctx->tower_in_idx = ULONG_MAX; + ctx->admin_in_idx = ULONG_MAX; + ctx->adopt_in_idx = ULONG_MAX; + ctx->adopt_out_idx = fd_topo_find_tile_out_link( topo, tile, "adopt_tower", 0UL ); + if( FD_LIKELY( ctx->adopt_out_idx!=ULONG_MAX ) ) { + fd_topo_link_t const * link = &topo->links[ tile->out_link_id[ ctx->adopt_out_idx ] ]; + ctx->adopt_out_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->adopt_out_chunk0 = fd_dcache_compact_chunk0( ctx->adopt_out_mem, link->dcache ); + ctx->adopt_out_wmark = fd_dcache_compact_wmark ( ctx->adopt_out_mem, link->dcache, link->mtu ); + ctx->adopt_out_chunk = ctx->adopt_out_chunk0; + } + ctx->admin_out_idx = fd_topo_find_tile_out_link( topo, tile, "failov_admin", 0UL ); + if( FD_LIKELY( ctx->admin_out_idx!=ULONG_MAX ) ) { + fd_topo_link_t const * link = &topo->links[ tile->out_link_id[ ctx->admin_out_idx ] ]; + ctx->admin_out_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->admin_out_chunk0 = fd_dcache_compact_chunk0( ctx->admin_out_mem, link->dcache ); + ctx->admin_out_wmark = fd_dcache_compact_wmark ( ctx->admin_out_mem, link->dcache, link->mtu ); + ctx->admin_out_chunk = ctx->admin_out_chunk0; + } + for( ulong i=0UL; iin_cnt; i++ ) { + fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ]; + if( FD_LIKELY( !strcmp( link->name, "gossip_ciseen" ) ) ) { + ctx->gossip_in_idx = i; + ctx->gossip_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->gossip_in_chunk0 = fd_dcache_compact_chunk0( ctx->gossip_in_mem, link->dcache ); + ctx->gossip_in_wmark = fd_dcache_compact_wmark ( ctx->gossip_in_mem, link->dcache, link->mtu ); + ctx->gossip_in_mtu = link->mtu; + continue; + } + if( FD_LIKELY( !strcmp( link->name, "tower_adopt" ) ) ) { + ctx->adopt_in_idx = i; + ctx->adopt_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->adopt_in_chunk0 = fd_dcache_compact_chunk0( ctx->adopt_in_mem, link->dcache ); + ctx->adopt_in_wmark = fd_dcache_compact_wmark ( ctx->adopt_in_mem, link->dcache, link->mtu ); + continue; + } + if( FD_LIKELY( !strcmp( link->name, "admin_failov" ) ) ) { + ctx->admin_in_idx = i; + ctx->admin_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->admin_in_chunk0 = fd_dcache_compact_chunk0( ctx->admin_in_mem, link->dcache ); + ctx->admin_in_wmark = fd_dcache_compact_wmark ( ctx->admin_in_mem, link->dcache, link->mtu ); + continue; + } + if( FD_LIKELY( !strcmp( link->name, "tower_out" ) ) ) { + ctx->tower_in_idx = i; + ctx->tower_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->tower_in_chunk0 = fd_dcache_compact_chunk0( ctx->tower_in_mem, link->dcache ); + ctx->tower_in_wmark = fd_dcache_compact_wmark ( ctx->tower_in_mem, link->dcache, link->mtu ); + continue; + } + /* The keyguard client reads sign_failov itself. */ + if( FD_LIKELY( !strcmp( link->name, "sign_failov" ) ) ) continue; + FD_LOG_ERR(( "unexpected input link name %s", link->name )); + } + + ulong sign_in_idx = fd_topo_find_tile_in_link ( topo, tile, "sign_failov", tile->kind_id ); + ulong sign_out_idx = fd_topo_find_tile_out_link( topo, tile, "failov_sign", tile->kind_id ); + FD_TEST( sign_in_idx!=ULONG_MAX && sign_out_idx!=ULONG_MAX ); + fd_topo_link_t const * sign_in = &topo->links[ tile->in_link_id [ sign_in_idx ] ]; + fd_topo_link_t const * sign_out = &topo->links[ tile->out_link_id[ sign_out_idx ] ]; + + fd_sleep_t * sleep = NULL; + if( FD_UNLIKELY( topo->sleep_obj_id!=ULONG_MAX ) ) { + sleep = fd_sleep_join( fd_topo_obj_laddr( topo, topo->sleep_obj_id ) ); + FD_TEST( sleep ); + } + + if( FD_UNLIKELY( !fd_keyguard_client_join( fd_keyguard_client_new( ctx->keyguard_client, + sign_out->mcache, + sign_out->dcache, + sign_in->mcache, + sign_in->dcache, + sign_out->mtu, + sign_in->mtu, + sleep, + sign_out->id, + fd_topo_find_link_consumer( topo, sign_out ) ) ) ) ) { + FD_LOG_ERR(( "failed to construct keyguard" )); + } +} + +/* Asks the sign tile to sign our member certificate with the staked key, + and installs it. */ +static void +request_member_cert( fd_failover_tile_ctx_t * ctx ) { + uchar msg [ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ]; + uchar cert[ 64 ]; + fd_failover_member_cert_msg( msg, ctx->hello.junk_pubkey ); + fd_keyguard_client_sign( ctx->keyguard_client, cert, msg, sizeof(msg), FD_KEYGUARD_SIGN_TYPE_ED25519 ); + if( FD_UNLIKELY( fd_failover_channel_set_member_cert( ctx->channel, cert ) ) ) { + FD_LOG_ERR(( "the sign tile signed our member certificate with a key other than the staked [paths.identity_key]" )); + } + ctx->member_cert_set = 1; + FD_LOG_NOTICE(( "the sign tile signed our member certificate with the staked identity, this machine can pair with the other failover machine now" )); +} + +/* Updates our session state when the channel's session changes. */ +static void +sync_session( fd_failover_tile_ctx_t * ctx ) { + ulong state = fd_failover_channel_state( ctx->channel ); + ulong generation = fd_failover_channel_generation( ctx->channel ); + if( FD_LIKELY( state==ctx->session.state && generation==ctx->session.generation ) ) return; + ctx->session.state = state; + ctx->session.generation = generation; + ctx->request.sent = 0; + ctx->session.close_after_send = 0; + if( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_HANDOFF_RESULT ) ctx->tx.valid = 0; + if( FD_UNLIKELY( state==FD_FAILOVER_SESSION_PAIRED ) ) { + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + ctx->session.peer_boot_id = peer->boot_id; + ctx->session.peer_role = peer->role; + if( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_SENT ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_PENDING; + if( FD_UNLIKELY( ( ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY || + ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ) && + peer->role==(uchar)FD_FAILOVER_ROLE_ACTIVE ) ) ctx->promotion.active_seen = 1; + if( FD_UNLIKELY( peer->role==(uchar)FD_FAILOVER_ROLE_ACTIVE && ( ctx->peer_tower.valid ) ) ) + FD_LOG_NOTICE(( "peer boot %016lx authenticated ACTIVE, discarding saved final state from before its voting tenure, keeping coverage floors", peer->boot_id )); + if( FD_UNLIKELY( peer->role==(uchar)FD_FAILOVER_ROLE_ACTIVE ) ) { + ctx->peer_tower.valid = 0; + } + } +} + +/* Returns the port a handoff dials, from `failover promote --port` if + given, else our own. */ +static inline ushort +dial_port( fd_failover_tile_ctx_t const * ctx ) { + return ctx->cmd_port ? ctx->cmd_port : ctx->port; +} + +static void +set_role( fd_failover_tile_ctx_t * ctx, + ulong role ) { + ctx->role = role; + ctx->hello.role = (uchar)role; + fd_failover_channel_set_role( ctx->channel, role ); +} + +/* Sets the replay slot past which the current step is overdue. */ +static void +deadline_start( fd_failover_tile_ctx_t * ctx, + ulong slots ) { + ctx->deadline_slot = ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) + ? FD_FAILOVER_SLOT_NULL + : fd_ulong_sat_add( ctx->replay_slot, slots ); +} + +/* Sets the slot deadline to slots past the current replay slot, unless + it is already set or no replay slot has been seen yet. */ +static void +deadline_arm( fd_failover_tile_ctx_t * ctx, + ulong slots ) { + if( FD_LIKELY( ctx->deadline_slot!=FD_FAILOVER_SLOT_NULL ) ) return; + if( FD_UNLIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) ) return; + ctx->deadline_slot = fd_ulong_sat_add( ctx->replay_slot, slots ); +} + +static int +deadline_expired( fd_failover_tile_ctx_t const * ctx ) { + return ctx->deadline_slot!=FD_FAILOVER_SLOT_NULL && + ctx->replay_slot !=FD_FAILOVER_SLOT_NULL && + ctx->replay_slot>ctx->deadline_slot; +} + +/* Sets the promotion deadline, in slots and on our clock, with room for + the floor slack when adopting the vote account. */ +static void +promote_deadline_arm( fd_failover_tile_ctx_t * ctx, + long now ) { + deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + if( FD_UNLIKELY( ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT && + ctx->promotion.floor!=FD_FAILOVER_SLOT_NULL && + ctx->deadline_slot!=FD_FAILOVER_SLOT_NULL ) ) { + ctx->deadline_slot = fd_ulong_max( ctx->deadline_slot, + fd_ulong_sat_add( ctx->promotion.floor, FD_FAILOVER_FLOOR_SLACK_SLOTS+FD_FAILOVER_DEADLINE_SLOTS ) ); + } + if( FD_UNLIKELY( !ctx->promotion.until ) ) { + ulong slots = FD_FAILOVER_DEADLINE_SLOTS; + if( FD_UNLIKELY( ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT && + ctx->promotion.floor!=FD_FAILOVER_SLOT_NULL ) ) slots += FD_FAILOVER_FLOOR_SLACK_SLOTS; + ctx->promotion.until = fd_long_sat_add( now, (long)slots*FD_FAILOVER_DEADLINE_SLOT_NANOS ); + } +} + +/* Returns whether the promotion passed its slot deadline or its clock + deadline. */ +static int +promote_expired( fd_failover_tile_ctx_t const * ctx, + long now ) { + if( FD_LIKELY( nowpromotion.until ) ) return deadline_expired( ctx ); + if( FD_UNLIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) ) { + FD_LOG_WARNING(( "%s: the promotion ran out of time before replay started, wait until this machine has caught up", ctx->promotion.label )); + } else { + FD_LOG_WARNING(( "%s: the promotion ran out of time with replay at slot %lu, replay may be stuck, check that this machine is catching up", ctx->promotion.label, ctx->replay_slot )); + } + return 1; +} + +/* What the operator knows our demotion as, a handoff when a peer asked + for it, else a demotion. */ +static inline char const * +demotion_kind( fd_failover_tile_ctx_t const * ctx ) { + return ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ? "handoff" : "demotion"; +} + +/* Marks the identity switch in flight as overdue, flags stuck and warns + once. We keep waiting for the admin tile's answer. */ +static void +switch_overdue( fd_failover_tile_ctx_t * ctx ) { + ctx->stuck = 1; + if( FD_LIKELY( ctx->id_switch.overdue ) ) return; + ctx->id_switch.overdue = 1; + char demotion[ 48 ]; + fd_cstr_printf( demotion, sizeof(demotion), NULL, "%s %lu", demotion_kind( ctx ), ctx->handoff.id ); + FD_LOG_WARNING(( "%s: identity switch %lu to %s is overdue, its outcome is unknown, we keep waiting and refuse another switch, check the admin and sign tile logs", + ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? ctx->promotion.label : demotion, + ctx->id_switch.request_id, ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? "staked" : "junk" )); +} + +/* Only one control message can be outstanding at a time. Its recipient + is fixed even if the connection changes before it is sent. */ +static int +queue_control( fd_failover_tile_ctx_t * ctx, + ushort type, + fd_failover_peer_t const * to, + void const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( ctx->tx.valid || !payload_sz || payload_sz>sizeof(ctx->tx.payload) ) ) return -1; + ctx->tx.type = type; + ctx->tx.sz = (ushort)payload_sz; + ctx->tx.valid = 1; + ctx->tx.to = *to; + fd_memcpy( ctx->tx.payload, payload, payload_sz ); + return 0; +} + +/* Our DEMOTED goes to the handoff target. */ +static int +queue_demoted( fd_failover_tile_ctx_t * ctx ) { + return queue_control( ctx, (ushort)FD_FAILOVER_MSG_DEMOTED, &ctx->handoff.target, ctx->handoff.payload, ctx->handoff.payload_sz ); +} + +/* Tells the requester, whose member and boot are in to, that we recorded + its answer to our DEMOTED or that its handoff cannot go ahead. */ +static void +handoff_result_to( fd_failover_tile_ctx_t * ctx, + fd_failover_peer_t const * to, + ulong id, + ulong result ) { + fd_failover_handoff_result_t response = { .handoff_id=id, .result=result }; + if( FD_LIKELY( !queue_control( ctx, FD_FAILOVER_MSG_HANDOFF_RESULT, to, &response, sizeof(response) ) ) ) { + /* Keep the connection until the result has drained. */ + ctx->session.close_after_send = 1; + ctx->session.close_handoff_id = id; + } +} + +/* queue_reply queues our last response to a DEMOTED. If the control slot + is busy the response stays pending and step_controller tries again once + the slot drains. */ +static void +queue_reply( fd_failover_tile_ctx_t * ctx ) { + uchar payload[ sizeof(fd_failover_promote_rejected_t) ]; + ulong payload_sz = ctx->reply.type==(ushort)FD_FAILOVER_MSG_PROMOTE_ACK + ? fd_failover_promote_ack_encode( payload, ctx->reply.handoff_id ) + : fd_failover_promote_rejected_encode( payload, ctx->reply.handoff_id, ctx->reply.reason ); + ctx->reply.state = queue_control( ctx, ctx->reply.type, &ctx->reply.peer, payload, payload_sz ) ? FD_FAILOVER_REPLY_PENDING : FD_FAILOVER_REPLY_KEPT; +} + +/* Remembers our response to the DEMOTED with handoff_id from peer, + then queues it. */ +static void +finish_reply( fd_failover_tile_ctx_t * ctx, + fd_failover_peer_t const * peer, + ulong handoff_id, + ushort type, + uchar reason ) { + if( ctx->reply.state==FD_FAILOVER_REPLY_NONE || ctx->reply.peer.boot_id!=peer->boot_id || ctx->reply.handoff_id!=handoff_id || ctx->reply.type!=type ) + ctx->reply.log_at = 0L; + ctx->reply.peer = *peer; + ctx->reply.handoff_id = handoff_id; + ctx->reply.type = type; + ctx->reply.reason = reason; + queue_reply( ctx ); +} + +/* The name of a result the old active sent us for our handoff request, or + the one we sent, and what the operator can do about it. */ +static char const * +relayed_refusal( ulong result, + char const ** hint ) { + switch( result ) { + case FD_ADMINCTL_RESULT_SUCCESS: + *hint = "the old active recorded that we took the identity but this machine is a standby, check `failover status` on both machines"; + return "SUCCESS"; + case FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY: + *hint = "the active could not finish this handoff and may have given up the identity, check `failover status` there"; + return "PEER_UNREADY"; + case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: + *hint = "the active is busy with another transition, run `failover promote` here again once `failover status` there shows it idle"; + return "IN_PROGRESS"; + case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: + *hint = "the machine we dialed is not the active any more, check `failover status` on both machines"; + return "BAD_ROLE"; + case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER: + *hint = "the active has no eligible final vote state to hand over and keeps the identity, its log says why"; + return "NO_FINAL_TOWER"; + case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: + *hint = "this machine's replay is too far behind the active's last vote and the active keeps the identity, run `failover promote` here again once this machine has caught up"; + return "REPLAY_BEHIND"; + default: + *hint = "the active answered with a result this machine does not know, run the same Firedancer version on both machines"; + return "UNKNOWN"; + } +} + +/* What a control message we just sent changes on our side. */ +static void +control_sent( fd_failover_tile_ctx_t * ctx, + long now ) { + switch( ctx->tx.type ) { + case (ushort)FD_FAILOVER_MSG_DEMOTED: + if( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_PENDING ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_SENT; + if( now>=ctx->handoff.log_at ) { + ctx->handoff.log_at = fd_long_sat_add( now, 60000000000L ); + FD_LOG_NOTICE(( "handoff %lu: sent final state (DEMOTED) to peer boot %016lx, last vote slot %lu, we remain standby waiting for its answer", + ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot )); + } + break; + case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK: + case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: + if( now>=ctx->reply.log_at ) { + ctx->reply.log_at = fd_long_sat_add( now, 60000000000L ); + FD_LOG_NOTICE(( "handoff %lu: sent %s to peer boot %016lx, %s, waiting for its final confirmation", + ctx->reply.handoff_id, ctx->tx.type==FD_FAILOVER_MSG_PROMOTE_ACK ? "ACK" : "refusal", + ctx->reply.peer.boot_id, ctx->role==FD_FAILOVER_ROLE_ACTIVE ? "we hold the staked identity" : "we remain standby" )); + } + break; + case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: { + fd_failover_handoff_result_t result; + fd_memcpy( &result, ctx->tx.payload, sizeof(result) ); + if( now>=ctx->handoff.result_log_at ) { + ctx->handoff.result_log_at = fd_long_sat_add( now, 60000000000L ); + char const * hint; + FD_LOG_NOTICE(( "handoff %lu: sent %s confirmation%s%s%s to peer boot %016lx, closing connection after queued bytes drain", + result.handoff_id, result.result==FD_ADMINCTL_RESULT_SUCCESS ? "success" : "refusal", + result.result==FD_ADMINCTL_RESULT_SUCCESS ? "" : " (", + result.result==FD_ADMINCTL_RESULT_SUCCESS ? "" : relayed_refusal( result.result, &hint ), + result.result==FD_ADMINCTL_RESULT_SUCCESS ? "" : ")", ctx->tx.to.boot_id )); + } + break; + } + default: break; + } +} + +static void +pending_flush( fd_failover_tile_ctx_t * ctx, + long now ) { + if( FD_LIKELY( !ctx->tx.valid ) ) return; + if( FD_UNLIKELY( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED || + fd_failover_channel_tx_pending( ctx->channel ) ) ) return; + /* We send a queued message only to the member and boot it was queued for. */ + if( FD_UNLIKELY( !peer_eq( &ctx->tx.to, ctx->session.peer_boot_id, fd_failover_channel_peer_hello( ctx->channel )->junk_pubkey ) ) ) return; + if( FD_UNLIKELY( fd_failover_channel_send( ctx->channel, now, ctx->tx.type, ctx->tx.payload, ctx->tx.sz ) ) ) return; + control_sent( ctx, now ); + ctx->tx.valid = 0; +} + +/* Keep the tower of our latest vote from the tower's vote transaction. + It is the final tower we hand over at a demotion. A malformed + transaction is logged and skipped, the previous tower stays. */ +static void +prepare_consensus( fd_failover_tile_ctx_t * ctx, + fd_tower_slot_done_t const * done ) { + if( FD_UNLIKELY( !done->vote_txn_sz || done->vote_txn_sz>sizeof(done->vote_txn) ) ) { + FD_LOG_WARNING(( "tower produced an invalid vote transaction size" )); + return; + } + + uchar txn_mem[ FD_TXN_MAX_SZ ] __attribute__((aligned(alignof(fd_txn_t)))); + fd_compact_tower_sync_serde_t serde; + if( FD_UNLIKELY( !fd_txn_parse( done->vote_txn, done->vote_txn_sz, txn_mem, NULL ) || + !fd_txn_parse_simple_vote( (fd_txn_t const *)txn_mem, done->vote_txn, &serde ) ) ) { + FD_LOG_WARNING(( "tower produced an invalid vote transaction" )); + return; + } + + fd_tower_vote_t votes[ FD_TOWER_VOTE_MAX ]; + ulong vote_cnt; + ulong root; + if( FD_UNLIKELY( fd_compact_tower_sync_to_votes( &serde, votes, &vote_cnt, &root ) || + !vote_cnt || votes[ vote_cnt-1UL ].slot!=done->vote_slot ) ) { + FD_LOG_WARNING(( "tower vote transaction does not match its slot metadata" )); + return; + } + + uchar state[ FD_FAILOVER_TOWER_STATE_MAX ]; + ulong state_sz = 0UL; + if( FD_UNLIKELY( fd_compact_tower_sync_ser( &serde, state, sizeof(state), &state_sz ) ) ) { + FD_LOG_WARNING(( "tower vote transaction exceeds the failover state limit" )); + return; + } + + fd_memcpy( ctx->current_tower.state, state, state_sz ); + ctx->current_tower.valid = 1; + ctx->current_tower.tip = done->vote_slot; + ctx->current_tower.sz = state_sz; + ctx->tower_gap = 0; /* anything skipped before this vote is superseded by it */ +} + +/* Fold one tower slot_done into the local slot view. The tower can + report slots out of order across forks, so the replay slot is a + running maximum. */ +static void +consume_slot_done( fd_failover_tile_ctx_t * ctx, + fd_tower_slot_done_t const * done ) { + if( FD_LIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || done->replay_slot>ctx->replay_slot ) ) + ctx->replay_slot = done->replay_slot; + if( FD_LIKELY( done->root_slot!=FD_FAILOVER_SLOT_NULL ) ) ctx->root_slot = done->root_slot; + if( FD_LIKELY( done->has_vote_txn && done->vote_slot!=FD_FAILOVER_SLOT_NULL ) ) { + ctx->last_vote_slot = done->vote_slot; + /* Admin can unhalt the new signer before its switch response reaches + this tile. Those first votes already belong to the new tenure. */ + if( FD_LIKELY( ctx->role==FD_FAILOVER_ROLE_ACTIVE || ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH ) ) { + /* Any later promotion has to cover the votes we signed. */ + if( FD_LIKELY( ctx->own_floor==FD_FAILOVER_SLOT_NULL || done->vote_slot>ctx->own_floor ) ) ctx->own_floor = done->vote_slot; + prepare_consensus( ctx, done ); + } + } +} + +/* Sends the tower we adopt to the tower tile, empty for the vote + account. The response comes back with the request id so we can tell a + late one apart. Returns the id, or ULONG_MAX if there was no request to send. */ +static ulong +publish_adopt_state( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + if( FD_UNLIKELY( ctx->adopt_out_idx==ULONG_MAX || + ( !ctx->promotion.adopt.sz && ctx->promotion.source!=FD_FAILOVER_SOURCE_VOTE_ACCOUNT ) ) ) return ULONG_MAX; + + if( FD_UNLIKELY( !++ctx->adopt_request_id ) ) ctx->adopt_request_id++; + fd_memcpy( fd_chunk_to_laddr( ctx->adopt_out_mem, ctx->adopt_out_chunk ), ctx->promotion.adopt.state, ctx->promotion.adopt.sz ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + ulong ctl = ctx->promotion.empty && ctx->promotion.force && !ctx->promotion.from_peer ? FD_TOWER_ADOPT_CTL_EMPTY : 0UL; + fd_stem_publish( stem, ctx->adopt_out_idx, ctx->adopt_request_id, ctx->adopt_out_chunk, ctx->promotion.adopt.sz, ctl, tspub, tspub ); + ctx->adopt_out_chunk = fd_dcache_compact_next( ctx->adopt_out_chunk, ctx->promotion.adopt.sz, ctx->adopt_out_chunk0, ctx->adopt_out_wmark ); + ctx->adopt_result_fresh = 0; + return ctx->adopt_request_id; +} + +/* Asks the admin tile to switch to the given key. Only the public key + goes out, the signing tile preloaded both keypairs. Only one request + can be in flight, and the reply comes back with the same id so a late + reply can be told apart. */ +static ulong +request_switch( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem, + ulong key ) { + if( FD_UNLIKELY( ctx->admin_out_idx==ULONG_MAX || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) return ULONG_MAX; + + if( FD_UNLIKELY( !++ctx->id_switch.request_id ) ) ctx->id_switch.request_id++; + fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->admin_out_mem, ctx->admin_out_chunk ); + fd_memset( out, 0, sizeof(*out) ); + out->nonce = ctx->id_switch.request_id; + fd_failover_switch_req_t req; + fd_memcpy( req.identity, key==FD_FAILOVER_SWITCH_KEY_STAKED ? ctx->hello.staked_pubkey : ctx->hello.junk_pubkey, 32UL ); + fd_memcpy( out->payload, &req, sizeof(req) ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_SWITCH_REQ, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub ); + ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark ); + ctx->id_switch.pending_key = key; + ctx->id_switch.fresh = 0; + return ctx->id_switch.request_id; +} + +/* Records the result of the switch we asked for. A reply for some other + request is dropped, otherwise an old reply could be mistaken for a + finished demotion. */ +static int +switch_answer( fd_failover_tile_ctx_t * ctx, + ulong nonce ) { + if( FD_UNLIKELY( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT || + nonce!=ctx->id_switch.request_id ) ) { + FD_LOG_WARNING(( "dropping a stale identity switch answer" )); + return 0; + } + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT; + ctx->id_switch.fresh = 1; + return 1; +} + +/* Our DEMOTED got its response or cannot get one any more. */ +static void +handoff_resolved( fd_failover_tile_ctx_t * ctx, + ulong result ) { + if( FD_UNLIKELY( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ) ) ctx->tx.valid = 0; + if( FD_LIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) ctx->action = FD_FAILOVER_ACTION_IDLE; + ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE; + ctx->handoff.result = result; +} + +/* A demotion that ends without sending final state. Nobody can + promote on it, and a requester bound to it is told at once rather than + at its deadline. */ +static void +demotion_abort( fd_failover_tile_ctx_t * ctx ) { + if( FD_UNLIKELY( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) ) { + handoff_result_to( ctx, &ctx->handoff.target, ctx->handoff.id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + } + ctx->action = FD_FAILOVER_ACTION_IDLE; + ctx->stuck = 1; + ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE; +} + +/* Each demotion step gets a full deadline, in slots and on our clock. */ +static void +demotion_deadline_start( fd_failover_tile_ctx_t * ctx ) { + deadline_start( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + ctx->handoff.until = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_CHANNEL_IDLE_NANOS ); +} + +/* Give up the staked identity. The switch itself is asked for from + step_controller, after the command's response went out. A handoff + sends DEMOTED to the peer boot we see now, a demote sends nothing. */ +static void +start_demotion( fd_failover_tile_ctx_t * ctx, + int handoff ) { + int send = handoff && fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED; + fd_memset( &ctx->handoff, 0, sizeof(ctx->handoff) ); + ctx->handoff.id = ctx->handoff_base+(++ctx->handoff_cnt); + ctx->handoff.result = FD_FAILOVER_HANDOFF_NONE; /* until its DEMOTED goes out */ + ctx->handoff.delivery = send ? FD_FAILOVER_DELIVERY_PENDING : FD_FAILOVER_DELIVERY_NONE; + if( send ) { + ctx->handoff.target.boot_id = ctx->session.peer_boot_id; + fd_memcpy( ctx->handoff.target.junk, fd_failover_channel_peer_hello( ctx->channel )->junk_pubkey, 32UL ); + } + ctx->last_requested = 0; + ctx->stuck = 0; + if( FD_UNLIKELY( !send ) ) FD_LOG_NOTICE(( "demotion %lu started without a handoff, giving up the staked identity, no DEMOTED will be sent", ctx->handoff.id )); + demotion_deadline_start( ctx ); + ctx->action = FD_FAILOVER_ACTION_DEMOTE_SWITCH; +} + +/* The final tower has to be the tower of our last vote, with no tower + frag skipped since it was built. */ +static int +final_tower_ok( fd_failover_tile_ctx_t const * ctx ) { + return ctx->current_tower.valid && ctx->current_tower.sz && + ctx->last_vote_slot!=FD_FAILOVER_SLOT_NULL && + ctx->current_tower.tip==ctx->last_vote_slot && !ctx->tower_gap; +} + +/* Called once the junk key is installed everywhere. Our latest tower is + the final one. A bound handoff hands it to the peer. A standalone + demote requires shared-source recovery. */ +static void +demotion_switched( fd_failover_tile_ctx_t * ctx ) { + set_role( ctx, FD_FAILOVER_ROLE_STANDBY ); + FD_LOG_NOTICE(( "%s %lu: the tower drained, we are a standby now", demotion_kind( ctx ), ctx->handoff.id )); + + int final_ok = final_tower_ok( ctx ); + + if( FD_UNLIKELY( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_NONE ) ) { + /* Either member may recover after a standalone demote. There is no + bound exchange to tell us whether the peer has since voted. */ + ctx->peer_tower.valid = 0; + if( FD_UNLIKELY( !final_ok ) ) { + /* Without final state, recovery needs the vote account. */ + FD_LOG_WARNING(( "demotion %lu has no final tower for its last vote %lu, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->last_vote_slot )); + demotion_abort( ctx ); + return; + } + FD_LOG_NOTICE(( "demotion %lu is done, neither machine holds the staked identity now, nothing votes until `failover promote --force` runs on one of them", ctx->handoff.id )); + ctx->action = FD_FAILOVER_ACTION_IDLE; + return; + } + + ctx->handoff.payload_sz = final_ok + ? fd_failover_demoted_encode( ctx->handoff.payload, ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot, + ctx->current_tower.state, ctx->current_tower.sz ) + : 0UL; + if( FD_UNLIKELY( !ctx->handoff.payload_sz ) ) { + /* We have no final tower to hand over, or the one we have is not the + tower of our last vote. The peer gets no final state and cannot + promote. */ + FD_LOG_WARNING(( "handoff %lu has no final tower for its last vote %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->last_vote_slot )); + demotion_abort( ctx ); + return; + } + + ctx->handoff.result = FD_FAILOVER_HANDOFF_PENDING; + /* The control slot may still be holding an unsent response, so if this + fails we retry from the wait. */ + (void)queue_demoted( ctx ); + ctx->action = FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK; + demotion_deadline_start( ctx ); +} + +/* Returns the name of a refusal reason for the log. */ +static char const * +reject_name( uint reason ) { + static char const * const names[ FD_FAILOVER_REJECT_CNT ] = { + "none", "busy", "holds identity", "replay behind", + "adoption mismatch", "adoption failed", "switch failed" + }; + return reasonrequest.until ) ) fd_failover_channel_init_dialer( ctx->channel, ctx->request.addr, dial_port( ctx ) ); + ctx->action = FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT; + ctx->request.until = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_CHANNEL_IDLE_NANOS ); +} + +/* Stand back down. If the peer's DEMOTED asked for the promotion, we + tell it why and drop the tower it handed us, because after a refusal + either machine may recover on its own without reporting its later + votes. */ +static void +reject_promotion( fd_failover_tile_ctx_t * ctx, + uchar reason, + int bad_tower ) { + if( FD_UNLIKELY( bad_tower && ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ) ctx->peer_tower.valid = 0; + ctx->stuck = 1; + ctx->action = FD_FAILOVER_ACTION_IDLE; + if( FD_UNLIKELY( !ctx->promotion.from_peer ) ) { + FD_LOG_WARNING(( "%s: promotion failed (%s), we stay a standby, check that one machine votes and see `failover status`, " + "run `failover promote --force` again once this machine has caught up and the other cannot sign", ctx->promotion.label, reject_name( reason ) )); + return; + } + ctx->peer_tower.valid = 0; + ctx->promotion.from_peer = 0; + request_wait_result( ctx ); + if( FD_UNLIKELY( reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { + FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), we stay a standby, another machine may hold the staked identity, " + "check `failover status` on both machines", ctx->promotion.handoff_id, reject_name( reason ) )); + } else { + FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), we stay a standby and the peer already gave up the staked identity, " + "nothing votes until `failover promote --force` runs on one machine, check `failover status` on both machines first", + ctx->promotion.handoff_id, reject_name( reason ) )); + } + finish_reply( ctx, &ctx->promotion.peer, ctx->promotion.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason ); +} + +/* The coverage floor is the higher of the peer's reported last vote and our highest signed vote. */ +static ulong +coverage_floor( fd_failover_tile_ctx_t const * ctx ) { + if( FD_UNLIKELY( ctx->peer_floor==FD_FAILOVER_SLOT_NULL ) ) return ctx->own_floor; + if( FD_UNLIKELY( ctx->own_floor ==FD_FAILOVER_SLOT_NULL ) ) return ctx->peer_floor; + return fd_ulong_max( ctx->peer_floor, ctx->own_floor ); +} + +/* Take the staked identity with the tower from source. A peer is given + when that member's DEMOTED with handoff_id asked for it, it gets our + response. An operator promotion passes NULL and 0. force is promote + --force. */ +static void +start_promotion( fd_failover_tile_ctx_t * ctx, + ulong source, + fd_failover_peer_t const * peer, + ulong handoff_id, + int force ) { + int from_peer = !!peer; + fd_failover_peer_t bound = {0}; + if( from_peer ) bound = *peer; + fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) ); + if( source==FD_FAILOVER_SOURCE_PEER ) ctx->promotion.adopt = ctx->peer_tower; + else ctx->promotion.adopt.tip = FD_FAILOVER_SLOT_NULL; + if( from_peer ) { + ctx->promotion.peer = bound; + ctx->promotion.handoff_id = handoff_id; + fd_cstr_printf( ctx->promotion.label, sizeof(ctx->promotion.label), NULL, "handoff %lu", handoff_id ); + } else { + fd_cstr_printf( ctx->promotion.label, sizeof(ctx->promotion.label), NULL, "operator promotion" ); + } + ctx->promotion.source = source; + ctx->promotion.from_peer = from_peer; + ctx->promotion.force = force; + ctx->promotion.floor = coverage_floor( ctx ); + /* Until this tenure votes, its final tower is exactly the adopted one, + not a tower cached from an earlier tenure. */ + ctx->current_tower = ctx->promotion.adopt; + ctx->tower_gap = 0; + ctx->last_vote_slot = ctx->promotion.adopt.tip; + ctx->stuck = 0; + deadline_start( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY; + if( FD_UNLIKELY( from_peer && ctx->promotion.adopt.tip!=FD_FAILOVER_SLOT_NULL && + ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || ctx->replay_slotpromotion.adopt.tip ) ) ) { + FD_LOG_NOTICE(( "%s: the promotion waits for replay to reach slot %lu before it adopts %s", ctx->promotion.label, ctx->promotion.adopt.tip, source_name( source ) )); + } +} + +/* An operator promotion tries the stored peer tower, then the vote + account, then with --force an empty tower. We enter only before the + adoption request goes out or after its response, so we do not overlap + an adoption. A handoff adopts its bound final state or refuses. */ +static int +promote_fallback( fd_failover_tile_ctx_t * ctx, + char const * reason ) { + if( FD_UNLIKELY( ctx->promotion.from_peer || ctx->promotion.empty ) ) return 0; + if( ctx->promotion.source!=FD_FAILOVER_SOURCE_VOTE_ACCOUNT ) { + FD_LOG_NOTICE(( "%s: %s is not eligible (%s), trying the vote account", ctx->promotion.label, + source_name( ctx->promotion.source ), reason )); + if( ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ctx->peer_tower.valid = 0; + ctx->promotion.source = FD_FAILOVER_SOURCE_VOTE_ACCOUNT; + } else { + FD_LOG_WARNING(( "%s: the vote account cannot be adopted (%s), --force proceeds with an empty tower", ctx->promotion.label, reason )); + ctx->promotion.empty = 1; + } + ctx->promotion.adopt.valid = 0; + ctx->promotion.adopt.tip = FD_FAILOVER_SLOT_NULL; + ctx->promotion.adopt.sz = 0UL; + ctx->current_tower = ctx->promotion.adopt; + ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; + ctx->promotion.retry = 0; + ctx->adopt_result_fresh = 0; + ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY; + return 1; +} + +/* Returns whether the adopted tower reaches the coverage floor, the + highest vote the peer reported or we signed. The vote account also + passes once replay is past the floor by the slack. */ +static int +floor_covered( fd_failover_tile_ctx_t const * ctx ) { + ulong floor = ctx->promotion.floor; + ulong tip = ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT ? ctx->adopt_result.acct_vote_slot + : ctx->adopt_result.vote_slot; + if( FD_LIKELY( floor==FD_FAILOVER_SLOT_NULL ) ) return 1; + if( FD_LIKELY( tip!=FD_FAILOVER_SLOT_NULL && tip>=floor ) ) return 1; + return ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT && + ctx->replay_slot!=FD_FAILOVER_SLOT_NULL && + ctx->replay_slot>fd_ulong_sat_add( floor, FD_FAILOVER_FLOOR_SLACK_SLOTS ); +} + +/* Whether an active peer authenticated after this promotion started, + which stops it unless it is forced. */ +static int +promote_holder_seen( fd_failover_tile_ctx_t const * ctx ) { + if( FD_UNLIKELY( ctx->promotion.force ) ) return 0; + if( FD_UNLIKELY( ctx->promotion.active_seen ) ) return 1; + return 0; +} + +static void +step_demote_drain( fd_failover_tile_ctx_t * ctx ) { + deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + int expired = deadline_expired( ctx ) || ( ctx->handoff.until && fd_failover_clock()>=ctx->handoff.until ); + /* The watermark is the tower tile's output sequence when it stopped + signing. */ + if( FD_UNLIKELY( !fd_seq_ge( fd_seq_inc( ctx->tower_seen_seq, 1UL ), ctx->id_switch.result.tower_watermark ) ) ) { + if( FD_UNLIKELY( expired ) ) { + FD_LOG_WARNING(( "%s %lu: the tower did not report its last votes in time, up to seq %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", demotion_kind( ctx ), ctx->handoff.id, ctx->id_switch.result.tower_watermark )); + ctx->id_switch.fresh = 0; + ctx->id_switch.overdue = 0; + set_role( ctx, FD_FAILOVER_ROLE_STANDBY ); + demotion_abort( ctx ); + } + return; + } + ctx->id_switch.fresh = 0; + ctx->id_switch.overdue = 0; + demotion_switched( ctx ); +} + +static void +step_demote_switch( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + int expired = deadline_expired( ctx ) || ( ctx->handoff.until && fd_failover_clock()>=ctx->handoff.until ); + if( FD_UNLIKELY( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.fresh ) ) { + if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_JUNK )==ULONG_MAX ) ) { + /* The switch request did not go out and we still have the + identity, so we send no final state. */ + FD_LOG_WARNING(( "an identity switch could not be requested, remaining the active" )); + demotion_abort( ctx ); + return; + } + FD_LOG_NOTICE(( "%s %lu: asked the admin tile to install the junk identity", demotion_kind( ctx ), ctx->handoff.id )); + return; + } + if( FD_LIKELY( !ctx->id_switch.fresh ) ) { + /* No response yet, keep waiting. Past the deadline we raise stuck. */ + if( FD_UNLIKELY( expired ) ) switch_overdue( ctx ); + return; + } + if( FD_UNLIKELY( ctx->id_switch.result.result!=FD_FAILOVER_SWITCH_OK ) ) { + /* The switch failed and we still have the identity, so we send no + final state. */ + ctx->id_switch.fresh = 0; + ctx->id_switch.overdue = 0; + FD_LOG_WARNING(( "the identity switch for %s %lu failed (%s), we keep the staked identity and stay the active", demotion_kind( ctx ), ctx->handoff.id, + ctx->id_switch.result.result==FD_FAILOVER_SWITCH_ERR_DISABLED ? "failover disabled" : "unknown" )); + demotion_abort( ctx ); + return; + } + /* The junk key is installed everywhere. */ + FD_LOG_NOTICE(( "%s %lu: the junk identity is installed, waiting for the tower to drain up to seq %lu", demotion_kind( ctx ), ctx->handoff.id, ctx->id_switch.result.tower_watermark )); + ctx->action = FD_FAILOVER_ACTION_DEMOTE_DRAIN; + demotion_deadline_start( ctx ); + step_demote_drain( ctx ); +} + +static void +step_demote_wait_ack( fd_failover_tile_ctx_t * ctx ) { + /* If the peer goes quiet we do not undo the demotion, we only raise + the alarm. The identity is already gone from this machine. */ + deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + /* A new peer boot cannot respond to this DEMOTED. Its earlier boot may + already have taken the identity and voted before losing its ACK. */ + if( FD_UNLIKELY( fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED && + fd_memeq( fd_failover_channel_peer_hello( ctx->channel )->junk_pubkey, ctx->handoff.target.junk, 32UL ) && + ctx->session.peer_boot_id!=ctx->handoff.target.boot_id ) ) { + FD_LOG_WARNING(( "the peer restarted before it answered handoff %lu and may have voted before it restarted, neither machine holds the staked identity now, " + "nothing votes until `failover promote --force` runs on one of them", ctx->handoff.id )); + ctx->peer_tower.valid = 0; + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_RESTARTED ); + return; + } + /* A reconnect from the same member can finish its handoff. */ + if( FD_UNLIKELY( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_PENDING && !ctx->tx.valid ) ) { + (void)queue_demoted( ctx ); + } + if( FD_UNLIKELY( deadline_expired( ctx ) || ( ctx->handoff.until && fd_failover_clock()>=ctx->handoff.until ) ) ) { + if( FD_UNLIKELY( !ctx->stuck ) ) + FD_LOG_WARNING(( "the peer has not answered handoff %lu, we remain standby and resend final state when it reconnects, " + "if the peer shows `role: active` run `failover promote` there to finish this handoff, " + "if it shows `role: standby` nothing votes and `failover promote --force` on one machine takes the identity", ctx->handoff.id )); + ctx->stuck = 1; + } + return; +} + +static void +step_promote_wait_replay( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + long now = fd_failover_clock(); + promote_deadline_arm( ctx, now ); + if( FD_UNLIKELY( promote_expired( ctx, now ) ) ) { + reject_promotion( ctx, FD_FAILOVER_REJECT_REPLAY_BEHIND, 0 ); + return; + } + /* Wait for replay to reach the tower's tip before adopting, otherwise + the tower refers to blocks we have not seen yet. */ + if( FD_UNLIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) ) return; + if( FD_UNLIKELY( ctx->promotion.adopt.tip!=FD_FAILOVER_SLOT_NULL && ctx->replay_slotpromotion.adopt.tip ) ) { + (void)promote_fallback( ctx, "replay has not reached its tip" ); + return; + } + ulong id = publish_adopt_state( ctx, stem ); + if( FD_UNLIKELY( id==ULONG_MAX ) ) { + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); + return; + } + FD_LOG_NOTICE(( "%s: asking the tower tile to adopt %s", ctx->promotion.label, + ctx->promotion.empty ? "an empty tower authorized by --force" : source_name( ctx->promotion.source ) )); + ctx->promotion.adopt_id = id; + ctx->promotion.retry = 0; + ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT; + return; +} + +static void +step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + long now = fd_failover_clock(); + promote_deadline_arm( ctx, now ); + if( FD_UNLIKELY( promote_expired( ctx, now ) ) ) { + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); + return; + } + if( FD_UNLIKELY( ctx->promotion.retry ) ) { + /* The tower tile has not replayed every block the tower votes on, or + the adopted tip is short of the floor. */ + if( FD_LIKELY( ctx->replay_slot<=ctx->promotion.retry_slot ) ) return; + ulong id = publish_adopt_state( ctx, stem ); + if( FD_UNLIKELY( id==ULONG_MAX ) ) { + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); + return; + } + ctx->promotion.adopt_id = id; + ctx->promotion.retry = 0; + return; + } + if( FD_LIKELY( !ctx->adopt_result_fresh ) ) return; + ctx->adopt_result_fresh = 0; + if( FD_UNLIKELY( ctx->adopt_result_id!=ctx->promotion.adopt_id ) ) return; + if( FD_UNLIKELY( ctx->adopt_result.result==FD_TOWER_ADOPT_ERR_UNREPLAYED ) ) { + if( promote_fallback( ctx, "its blocks have not been replayed" ) ) return; + if( FD_UNLIKELY( !ctx->promotion.from_peer ) ) { + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); + return; + } + if( FD_UNLIKELY( !ctx->promotion.retry_logged ) ) { + FD_LOG_NOTICE(( "%s: the tower tile has not replayed every block the tower votes on, asking again as replay moves", ctx->promotion.label )); + ctx->promotion.retry_logged = 1; + } + ctx->promotion.retry = 1; + ctx->promotion.retry_slot = ctx->replay_slot; + return; + } + if( FD_UNLIKELY( ctx->adopt_result.result!=FD_TOWER_ADOPT_SUCCESS ) ) { + if( promote_fallback( ctx, adopt_err_name( ctx->adopt_result.result ) ) ) return; + FD_LOG_WARNING(( "%s: the tower tile refused %s (%s)", ctx->promotion.label, source_name( ctx->promotion.source ), adopt_err_name( ctx->adopt_result.result ) )); + ulong result = ctx->adopt_result.result; + reject_promotion( ctx, result==FD_TOWER_ADOPT_ERR_BLOCK_MISMATCH ? FD_FAILOVER_REJECT_ADOPTION_MISMATCH + : FD_FAILOVER_REJECT_ADOPTION_FAILED, + result==FD_TOWER_ADOPT_ERR_BLOCK_MISMATCH || result==FD_TOWER_ADOPT_ERR_DECODE || + result==FD_TOWER_ADOPT_ERR_INVALID ); + return; + } + if( FD_UNLIKELY( ctx->promotion.adopt.tip!=FD_FAILOVER_SLOT_NULL && ctx->adopt_result.vote_slot!=ctx->promotion.adopt.tip ) ) { + if( promote_fallback( ctx, "adoption did not retain its final vote" ) ) return; + FD_LOG_WARNING(( "%s: the adopted tower ends at slot %lu, the tower says %lu, our root has passed its last vote, so it is a mismatch", ctx->promotion.label, ctx->adopt_result.vote_slot, ctx->promotion.adopt.tip )); + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_MISMATCH, 1 ); + return; + } + int covered = floor_covered( ctx ); + int empty_account = ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT && ctx->adopt_result.acct_vote_slot==FD_FAILOVER_SLOT_NULL; + if( FD_UNLIKELY( !ctx->promotion.from_peer && ( !covered || empty_account ) ) ) { + if( ctx->promotion.source!=FD_FAILOVER_SOURCE_VOTE_ACCOUNT && + promote_fallback( ctx, "its votes do not cover the known signing history" ) ) return; + if( !ctx->promotion.empty ) { + char tip[ 32 ], floor[ 32 ]; + FD_LOG_WARNING(( "%s: --force accepts %s vote-account history (last account vote %s, required coverage floor %s), earlier votes and lockouts may be lost", ctx->promotion.label, + empty_account ? "empty" : "incomplete", slot_text( ctx->adopt_result.acct_vote_slot, tip ), slot_text( ctx->promotion.floor, floor ) )); + } + } + if( FD_UNLIKELY( !covered && !ctx->promotion.force ) ) { + /* Only a handoff gets here. The tip of the peer's tower does not + change, so we do not ask again and the deadline ends that wait. */ + if( FD_UNLIKELY( !ctx->promotion.floor_logged ) ) { + FD_LOG_WARNING(( "%s: %s ends at slot %lu, short of the coverage floor at slot %lu, the promotion stops at its deadline", ctx->promotion.label, + source_name( ctx->promotion.source ), ctx->adopt_result.vote_slot, ctx->promotion.floor )); + ctx->promotion.floor_logged = 1; + } + ctx->promotion.retry = 1; + ctx->promotion.retry_slot = FD_FAILOVER_SLOT_NULL; + return; + } + if( FD_UNLIKELY( promote_holder_seen( ctx ) ) ) { + /* The identity may be held elsewhere now, so we do not install it + too. */ + FD_LOG_WARNING(( "%s: another machine may hold the identity now, a peer authenticated as ACTIVE during this promotion, " + "not installing the staked key, check `failover status` on both machines", ctx->promotion.label )); + reject_promotion( ctx, FD_FAILOVER_REJECT_HOLDS_IDENTITY, 0 ); + return; + } + if( FD_UNLIKELY( ctx->promotion.adopt.tip==FD_FAILOVER_SLOT_NULL ) ) ctx->last_vote_slot = ctx->adopt_result.vote_slot; + if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_STAKED )==ULONG_MAX ) ) { + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); + return; + } + FD_LOG_NOTICE(( "%s: the tower tile adopted %s, asking the admin tile to install the staked identity", ctx->promotion.label, + ctx->promotion.empty ? "an empty tower authorized by --force" : source_name( ctx->promotion.source ) )); + ctx->action = FD_FAILOVER_ACTION_PROMOTE_SWITCH; + return; +} + +static void +step_promote_switch( fd_failover_tile_ctx_t * ctx ) { + deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + if( FD_LIKELY( !ctx->id_switch.fresh ) ) { + /* No response yet. Standing down here would tell the peer nobody + promoted while the staked key may well be installed. So we keep + waiting and raise stuck past the deadline, the late response still + counts. */ + if( FD_UNLIKELY( deadline_expired( ctx ) || ( ctx->promotion.until && fd_failover_clock()>=ctx->promotion.until ) ) ) switch_overdue( ctx ); + return; + } + ctx->id_switch.fresh = 0; + ctx->id_switch.overdue = 0; + if( FD_UNLIKELY( ctx->id_switch.result.result!=FD_FAILOVER_SWITCH_OK ) ) { + reject_promotion( ctx, FD_FAILOVER_REJECT_SWITCH_FAILED, 0 ); + return; + } + set_role( ctx, FD_FAILOVER_ROLE_ACTIVE ); + /* A new tenure starts, the peer tower we kept is older than its votes. */ + ctx->peer_tower.valid = 0; + ctx->taken = 0; + ctx->action = FD_FAILOVER_ACTION_IDLE; + ctx->stuck = 0; + if( FD_UNLIKELY( ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ) ) { + FD_LOG_NOTICE(( "%s: we are the active now, the staked identity is installed with %s", ctx->promotion.label, + ctx->promotion.empty ? "an empty tower authorized by --force" : "the vote account's tower" )); + } else { + FD_LOG_NOTICE(( "%s: we are the active now, the staked identity is installed and the adopted tower ends at slot %lu", ctx->promotion.label, ctx->last_vote_slot )); + } + if( FD_UNLIKELY( ctx->promotion.from_peer ) ) { + /* The ACK tells the old active its handoff is done. We keep it as + our response, so the same DEMOTED again gets the ACK again. */ + ctx->promotion.from_peer = 0; + request_wait_result( ctx ); + finish_reply( ctx, &ctx->promotion.peer, ctx->promotion.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_ACK, (uchar)FD_FAILOVER_REJECT_NONE ); + } + return; +} + +/* Advances the current failover step. */ +static void +step_controller( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + if( FD_UNLIKELY( ctx->reply.state==FD_FAILOVER_REPLY_PENDING && !ctx->tx.valid ) ) queue_reply( ctx ); + if( FD_LIKELY( ctx->action==FD_FAILOVER_ACTION_IDLE ) ) return; + + switch( ctx->action ) { + + case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER: + case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: + return; + + case FD_FAILOVER_ACTION_DEMOTE_SWITCH: + step_demote_switch( ctx, stem ); + return; + + case FD_FAILOVER_ACTION_DEMOTE_DRAIN: + step_demote_drain( ctx ); + return; + + case FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK: + step_demote_wait_ack( ctx ); + return; + + case FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY: + step_promote_wait_replay( ctx, stem ); + return; + + case FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT: + step_promote_wait_adopt( ctx, stem ); + return; + + case FD_FAILOVER_ACTION_PROMOTE_SWITCH: + step_promote_switch( ctx ); + return; + + default: FD_LOG_ERR(( "unexpected failover action %lu", ctx->action )); + } +} + +static void +request_end( fd_failover_tile_ctx_t * ctx, long now, ulong result ); + +/* Describes what our handoff request is waiting for, for logging. */ +static char const * +request_wait( fd_failover_tile_ctx_t const * ctx ) { + switch( ctx->action ) { + case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER: + if( !ctx->request.peer.boot_id ) return "no peer authenticated, no handoff request sent, we remain standby"; + if( !ctx->request.send_cnt ) return "peer authenticated, no handoff request sent, we remain standby"; + return "waiting for final state, the peer may still hold the identity"; + case FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY: return "replay catch-up continues locally, we remain standby"; + case FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT: return "adoption continues locally, we remain standby"; + case FD_FAILOVER_ACTION_PROMOTE_SWITCH: return "the staked key switch continues locally, its outcome is still unknown"; + case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: + return ctx->role==FD_FAILOVER_ROLE_ACTIVE ? "we hold the staked identity, waiting for the peer to confirm our ACK" + : "we remain standby, waiting for the peer to confirm our refusal"; + default: return "the local transition is retained"; + } +} + +/* What the operator can do about a paused request. */ +static char const * +request_next( fd_failover_tile_ctx_t const * ctx ) { + if( FD_UNLIKELY( ctx->request.peer_restarted ) ) return "the active boot we authenticated restarted, `failover promote` here starts a new handoff"; + switch( ctx->action ) { + case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER: + if( !ctx->request.send_cnt ) return "nothing was sent and the active is unchanged, `failover promote` here resumes the request, " + "and if the active is down and cannot sign `failover promote --force` takes over"; + return "the active may already have given up the identity, check `failover status` there, `failover promote` here resumes the request"; + case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: + if( ctx->role==FD_FAILOVER_ROLE_ACTIVE ) return "this machine holds the staked identity and votes, `failover promote --yes` here asks the old active again to confirm"; + if( ctx->reply.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) return "this machine refused the handoff because another machine may hold the staked identity, " + "check `failover status` on both machines, `failover promote` here asks the old active again to confirm"; + return "neither machine holds the staked identity, `failover promote` here asks the old active again to confirm, " + "and `failover promote --force` on one machine takes the identity"; + default: + return "the local promotion goes on and dials again when it ends"; + } +} + +static void +request_pause( fd_failover_tile_ctx_t * ctx, + long now, + char const * why ) { + FD_LOG_WARNING(( "handoff %lu paused (%s): %s, dialing stopped, %s", ctx->request.id, why, request_wait( ctx ), request_next( ctx ) )); + ctx->request.until = 0L; + ctx->stuck = 1; + fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); + fd_failover_channel_hangup( ctx->channel, now ); + sync_session( ctx ); +} + +/* handoff_result_to for the member and boot of the current session. */ +static void +handoff_result( fd_failover_tile_ctx_t * ctx, + ulong id, + ulong result ) { + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + fd_failover_peer_t to = { .boot_id=peer->boot_id }; + fd_memcpy( to.junk, peer->junk_pubkey, 32UL ); + handoff_result_to( ctx, &to, id, result ); +} + +/* An ACK or refusal counts only for the DEMOTED we sent, from the member + and boot it went to. */ +static int +demoted_answer_bound( fd_failover_tile_ctx_t const * ctx, + ulong handoff_id ) { + return ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE && handoff_id==ctx->handoff.id && + peer_eq( &ctx->handoff.target, ctx->session.peer_boot_id, fd_failover_channel_peer_hello( ctx->channel )->junk_pubkey ) && + ctx->action!=FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->action!=FD_FAILOVER_ACTION_DEMOTE_DRAIN; +} + +/* Handle a controller message. Anything malformed drops the session, + these messages hand off the staked identity so we do not guess at them. */ +static void +handle_control( fd_failover_tile_ctx_t * ctx, + ushort type, + ulong payload_sz, + long now ) { + switch( type ) { + + case (ushort)FD_FAILOVER_MSG_HANDOFF_REQUEST: { + fd_failover_handoff_request_t request; + if( FD_UNLIKELY( !fd_failover_handoff_request_decode( &request, ctx->rx, payload_sz ) ) ) break; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + if( FD_UNLIKELY( request.target_boot_id!=ctx->hello.boot_id ) ) { + handoff_result( ctx, request.handoff_id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + return; + } + if( FD_UNLIKELY( request.handoff_id==ctx->handoff.id && peer_eq( &ctx->handoff.target, peer->boot_id, peer->junk_pubkey ) ) ) { + if( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH || ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN ) return; + if( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) { + if( !ctx->tx.valid ) (void)queue_demoted( ctx ); + } else { + handoff_result( ctx, request.handoff_id, ctx->handoff.result==FD_FAILOVER_HANDOFF_TAKEN + ? FD_ADMINCTL_RESULT_SUCCESS : FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + } + return; + } + ulong result = FD_ADMINCTL_RESULT_SUCCESS; + if( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT || ctx->tx.valid || ctx->session.close_after_send ) result = FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; + else if( ctx->role!=FD_FAILOVER_ROLE_ACTIVE || peer->role!=FD_FAILOVER_ROLE_STANDBY ) result = FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; + else if( !final_tower_ok( ctx ) ) result = FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER; + /* A standby this far behind could not reach our last vote before its + promotion deadline. */ + else if( request.replay_slot==FD_FAILOVER_SLOT_NULL || + fd_ulong_sat_add( request.replay_slot, FD_FAILOVER_DEADLINE_SLOTS )last_vote_slot ) result = FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND; + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + if( result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER && now>=ctx->handoff_refuse_log_at ) { + ctx->handoff_refuse_log_at = fd_long_sat_add( now, 60000000000L ); + char tip[ 32 ], last[ 32 ]; + char const * why = ctx->tower_gap ? "a vote-history update was skipped, the final state cannot be verified" : + ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ? "no vote is known to the failover controller, it cannot determine whether this machine has not voted or is unable to vote" : + !ctx->current_tower.valid || !ctx->current_tower.sz ? "a vote is known but its final state is unavailable" : + "the saved final state does not match the latest known vote"; + FD_LOG_WARNING(( "handoff %lu refused: %s (latest known vote %s, saved history tip %s), this machine keeps the staked identity, check local voting progress and vote-history logs before retrying", + request.handoff_id, why, slot_text( ctx->last_vote_slot, last ), + slot_text( ctx->current_tower.valid ? ctx->current_tower.tip : FD_FAILOVER_SLOT_NULL, tip ) )); + } else if( result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND && now>=ctx->handoff_refuse_log_at ) { + ctx->handoff_refuse_log_at = fd_long_sat_add( now, 60000000000L ); + char replay[ 32 ]; + FD_LOG_WARNING(( "handoff %lu refused: the standby's replay is at slot %s, more than %lu slots behind our last vote at slot %lu, this machine keeps the staked identity, " + "run `failover promote` there again once it has caught up", + request.handoff_id, slot_text( request.replay_slot, replay ), FD_FAILOVER_DEADLINE_SLOTS, ctx->last_vote_slot )); + } + handoff_result( ctx, request.handoff_id, result ); + return; + } + start_demotion( ctx, 1 ); + ctx->handoff.id = request.handoff_id; + char member[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( peer->junk_pubkey, NULL, member ); + FD_LOG_NOTICE(( "handoff %lu: accepted request from member %s boot %016lx, preparing to give up the staked identity", ctx->handoff.id, member, ctx->handoff.target.boot_id )); + return; + } + + case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: { + fd_failover_handoff_result_t result; + if( FD_UNLIKELY( !fd_failover_handoff_result_decode( &result, ctx->rx, payload_sz ) ) ) break; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + if( FD_UNLIKELY( !ctx->request.id || result.handoff_id!=ctx->request.id || + !peer_eq( &ctx->request.peer, peer->boot_id, peer->junk_pubkey ) ) ) return; + if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && + ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) return; + int failed = result.result!=FD_ADMINCTL_RESULT_SUCCESS || ctx->role!=FD_FAILOVER_ROLE_ACTIVE; + if( FD_UNLIKELY( failed && ctx->role==FD_FAILOVER_ROLE_ACTIVE ) ) { + FD_LOG_WARNING(( "handoff %lu: the old active did not record that we took the identity and may hold it again, check `failover status` on both machines at once", result.handoff_id )); + } else if( FD_UNLIKELY( failed && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && ctx->reply.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { + FD_LOG_WARNING(( "handoff %lu ended, this machine refused the final state (%s) because another machine may hold the staked identity, and the old active " + "recorded it, check `failover status` on both machines", result.handoff_id, reject_name( ctx->reply.reason ) )); + } else if( FD_UNLIKELY( failed && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) { + FD_LOG_WARNING(( "handoff %lu ended, this machine refused the final state (%s) and the old active recorded it, neither machine holds the staked identity, " + "nothing votes until `failover promote --force` runs on one of them", result.handoff_id, reject_name( ctx->reply.reason ) )); + } else if( FD_UNLIKELY( failed ) ) { + char const * hint; + char const * name = relayed_refusal( result.result, &hint ); + FD_LOG_WARNING(( "handoff %lu ended (%s), %s", result.handoff_id, name, hint )); + } + request_end( ctx, now, failed ? FD_FAILOVER_HANDOFF_DECLINED : FD_FAILOVER_HANDOFF_TAKEN ); + if( !failed ) + FD_LOG_NOTICE(( "handoff %lu complete: the old active confirmed our ACK, we hold the staked identity, on-demand connection closed", result.handoff_id )); + else + FD_LOG_NOTICE(( "handoff %lu: request ended, %s, on-demand connection closed", result.handoff_id, + ctx->role==FD_FAILOVER_ROLE_ACTIVE ? "we still hold the staked identity" : "we remain standby" )); + return; + } + + case (ushort)FD_FAILOVER_MSG_DEMOTED: { + /* The peer says it can no longer sign and hands us its tower. */ + fd_failover_demoted_t demoted; + if( FD_UNLIKELY( !fd_failover_demoted_decode( &demoted, ctx->rx, payload_sz ) ) ) break; + ulong boot_id = ctx->session.peer_boot_id; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + /* Bind before deduplication. An unsolicited DEMOTED cannot change + this operation or replace its cached response. */ + if( FD_UNLIKELY( !ctx->request.id || ctx->request.id!=demoted.handoff_id || + !peer_eq( &ctx->request.peer, boot_id, peer->junk_pubkey ) ) ) return; + if( FD_UNLIKELY( demoted.target_boot_id!=ctx->hello.boot_id ) ) break; + /* The same DEMOTED again. Once we are done with it we repeat our + response. */ + if( FD_UNLIKELY( ctx->promotion.from_peer && ctx->promotion.peer.boot_id==boot_id && ctx->promotion.handoff_id==demoted.handoff_id ) ) return; + if( FD_UNLIKELY( ctx->reply.state!=FD_FAILOVER_REPLY_NONE && ctx->reply.handoff_id==demoted.handoff_id && peer_eq( &ctx->reply.peer, boot_id, peer->junk_pubkey ) ) ) { + queue_reply( ctx ); + return; + } + fd_failover_tower_t tower = { .valid=1, .tip=demoted.last_vote_slot, .sz=(ulong)demoted.state_len }; + fd_memcpy( tower.state, ctx->rx+sizeof(fd_failover_demoted_t), tower.sz ); + uchar reason = FD_FAILOVER_REJECT_NONE; + char const * why = NULL; + if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_STANDBY ) ) { + reason = FD_FAILOVER_REJECT_HOLDS_IDENTITY; + why = "we hold the staked identity"; + } else if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || + ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) { + reason = FD_FAILOVER_REJECT_BUSY; + why = "a transition or key switch is running here"; + } + if( FD_UNLIKELY( reason!=FD_FAILOVER_REJECT_NONE ) ) { + FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), %s, check `failover status` on both machines", demoted.handoff_id, reject_name( reason ), why )); + finish_reply( ctx, &ctx->request.peer, demoted.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason ); + return; + } + FD_LOG_NOTICE(( "handoff %lu: accepted the peer's final tower ending at slot %lu, waiting for local adoption before taking the identity", demoted.handoff_id, tower.tip )); + if( ctx->peer_floor==FD_FAILOVER_SLOT_NULL || tower.tip>ctx->peer_floor ) ctx->peer_floor = tower.tip; + ctx->peer_tower = tower; + ctx->session.peer_role = FD_FAILOVER_ROLE_STANDBY; + start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &ctx->request.peer, demoted.handoff_id, 0 ); + return; + } + + case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK: { + /* The peer took the identity, so our DEMOTED did its job. An ACK + after the deadline still counts, and we ignore an ACK we are not + waiting for rather than drop the session. */ + fd_failover_promote_ack_t ack; + if( FD_UNLIKELY( !fd_failover_promote_ack_decode( &ack, ctx->rx, payload_sz ) ) ) break; + if( FD_UNLIKELY( !demoted_answer_bound( ctx, ack.handoff_id ) ) ) return; + FD_LOG_NOTICE(( "handoff %lu: received ACK, the peer took the staked identity, we stay a standby", ack.handoff_id )); + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_TAKEN ); + ctx->taken = 1; + ctx->session.peer_role = FD_FAILOVER_ROLE_ACTIVE; + ctx->stuck = 0; + /* The peer votes past the final tower we handed it from here on. */ + ctx->peer_tower.valid = 0; + handoff_result( ctx, ack.handoff_id, FD_ADMINCTL_RESULT_SUCCESS ); + return; + } + + case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: { + /* The peer did not take the identity. Either machine may now recover + without telling the other, so we drop any tower the peer handed us + earlier. */ + fd_failover_promote_rejected_t rej; + if( FD_UNLIKELY( !fd_failover_promote_rejected_decode( &rej, ctx->rx, payload_sz ) ) ) break; + if( FD_UNLIKELY( !demoted_answer_bound( ctx, rej.handoff_id ) ) ) return; + if( FD_UNLIKELY( rej.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { + FD_LOG_WARNING(( "the peer declined handoff %lu (%s), it saw another machine that may hold the staked identity, this machine gave it up and " + "stays a standby, check `failover status` on both machines and the peer's log", rej.handoff_id, reject_name( rej.reason ) )); + } else { + FD_LOG_WARNING(( "the peer declined handoff %lu (%s), neither machine holds the staked identity now, nothing votes until `failover promote --force` runs on one of them, " + "see the peer's log first", rej.handoff_id, reject_name( rej.reason ) )); + } + ctx->peer_tower.valid = 0; + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_DECLINED ); + ctx->stuck = 1; + handoff_result( ctx, rej.handoff_id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + return; + } + + default: break; + } + + fd_failover_channel_protocol_error( ctx->channel, now ); + sync_session( ctx ); +} + +/* End our handoff request with result, an FD_FAILOVER_HANDOFF_*, and + hang up. */ +static void +request_end( fd_failover_tile_ctx_t * ctx, + long now, + ulong result ) { + int failed = result!=FD_FAILOVER_HANDOFF_TAKEN; + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ); + ctx->request.result = result; + ctx->request.id = 0UL; + ctx->request.until = 0L; + ctx->tx.valid = 0; + ctx->session.close_after_send = 0; + if( ctx->reply.state==FD_FAILOVER_REPLY_PENDING ) ctx->reply.state = FD_FAILOVER_REPLY_KEPT; + ctx->action = FD_FAILOVER_ACTION_IDLE; + ctx->stuck = failed; + fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); + fd_failover_channel_hangup( ctx->channel, now ); + sync_session( ctx ); +} + +/* The standby dials only while its operator's handoff is open. */ +static void +peer_poll( fd_failover_tile_ctx_t * ctx, + long now, + int * charge_busy ) { + sync_session( ctx ); + if( FD_UNLIKELY( ctx->request.id && ctx->request.until && now>=ctx->request.until && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || + ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) ) { + request_pause( ctx, now, !ctx->request.peer.boot_id ? "no peer authenticated before the deadline" : + !ctx->request.send_cnt ? "no handoff request sent before the deadline" : + "no final answer before the deadline" ); + return; + } + if( FD_UNLIKELY( ctx->session.close_after_send && !ctx->tx.valid && !fd_failover_channel_tx_pending( ctx->channel ) ) ) { + ctx->session.close_after_send = 0; + fd_failover_channel_hangup( ctx->channel, now ); + FD_LOG_NOTICE(( "handoff %lu: confirmation drained, on-demand connection closed, listener ready for the next request", ctx->session.close_handoff_id )); + sync_session( ctx ); + return; + } + + ushort type; + ulong payload_sz; + if( FD_UNLIKELY( fd_failover_channel_poll( ctx->channel, now, charge_busy, &type, ctx->rx, &payload_sz ) ) ) { + sync_session( ctx ); + handle_control( ctx, type, payload_sz, now ); + } + sync_session( ctx ); + if( FD_LIKELY( !ctx->request.id || !ctx->request.until || ctx->request.sent || + fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ) ) return; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + if( FD_UNLIKELY( !ctx->request.peer.boot_id ) ) { + if( FD_UNLIKELY( peer->role!=FD_FAILOVER_ROLE_ACTIVE ) ) { + FD_LOG_WARNING(( "handoff %lu dialed a standby at `" FD_IP4_ADDR_FMT ":%hu`, give the active's address with `failover promote --address`, " + "and if both machines show `role: standby` no machine holds the identity and `failover promote --force` on one of them takes it", + ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), dial_port( ctx ) )); + request_end( ctx, now, FD_FAILOVER_HANDOFF_NOT_ACTIVE ); + return; + } + ctx->request.peer.boot_id = peer->boot_id; + fd_memcpy( ctx->request.peer.junk, peer->junk_pubkey, 32UL ); + } else if( FD_UNLIKELY( !peer_eq( &ctx->request.peer, peer->boot_id, peer->junk_pubkey ) ) ) { + /* The peer restarted before it confirmed our answer, so it can no + longer confirm this handoff and the wait ends. */ + if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && + ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && + fd_memeq( peer->junk_pubkey, ctx->request.peer.junk, 32UL ) ) ) { + int active = ctx->role==FD_FAILOVER_ROLE_ACTIVE; + int peer_active = peer->role==FD_FAILOVER_ROLE_ACTIVE; + FD_LOG_WARNING(( "handoff %lu: the peer restarted before it confirmed our %s, ending the request, %s, %s", ctx->request.id, + active ? "ACK" : "refusal", active ? "we hold the staked identity" : "we remain standby", + active && peer_active ? "and its new boot says it is active too, check `failover status` on both machines at once" : + active ? "its new boot is a standby" : + peer_active ? "its new boot is the active" : + "its new boot is a standby too, nothing votes until `failover promote --force` runs on one machine" )); + request_end( ctx, now, active ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ); + return; + } + /* The same member on a new boot will not demote for it. */ + ctx->request.peer_restarted = ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && fd_memeq( peer->junk_pubkey, ctx->request.peer.junk, 32UL ); + request_pause( ctx, now, ctx->request.peer_restarted ? "the authenticated boot restarted" : "authenticated member or boot changed" ); + return; + } + fd_failover_handoff_request_t request = { .handoff_id=ctx->request.id, .target_boot_id=ctx->request.peer.boot_id, .replay_slot=ctx->replay_slot }; + if( FD_LIKELY( !fd_failover_channel_send( ctx->channel, now, FD_FAILOVER_MSG_HANDOFF_REQUEST, + (uchar const *)&request, sizeof(request) ) ) ) { + ctx->request.sent = 1; + ctx->request.send_cnt = fd_ulong_sat_add( ctx->request.send_cnt, 1UL ); + if( now>=ctx->request.log_at ) { + ctx->request.log_at = fd_long_sat_add( now, 60000000000L ); + char member[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( ctx->request.peer.junk, NULL, member ); + FD_LOG_NOTICE(( "handoff %lu: sent REQUEST to authenticated member %s boot %016lx, %s (send %lu)", + ctx->request.id, member, ctx->request.peer.boot_id, request_wait( ctx ), ctx->request.send_cnt )); + } + *charge_busy = 1; + } +} + + +/* Checks whether a promotion may go ahead now. Returns + FD_ADMINCTL_RESULT_SUCCESS or the refusal. */ +static ulong +promote_guard( fd_failover_tile_ctx_t const * ctx, + long now, + int force ) { + int paired = fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED; + + if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_STANDBY ) ) return FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; + if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK && !force ) ) return FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING; + int request_wait = ctx->request.id && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || + ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + if( FD_UNLIKELY( ( ctx->action!=FD_FAILOVER_ACTION_IDLE && ctx->action!=FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK && + !( force && request_wait ) ) || + ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; + if( FD_UNLIKELY( force ) ) return FD_ADMINCTL_RESULT_SUCCESS; + /* The peer took the identity in our last handoff. */ + if( FD_UNLIKELY( ctx->taken ) ) return FD_FAILOVER_CONTROL_RESULT_TAKEN; + if( FD_UNLIKELY( paired && fd_failover_channel_peer_hello( ctx->channel )->role==FD_FAILOVER_ROLE_ACTIVE ) ) + return FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE; + /* Another host is publishing the staked identity in gossip. */ + if( FD_UNLIKELY( ctx->staked_seen_at && now>=ctx->staked_seen_at && + now-ctx->staked_seen_ataction==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) return FD_FAILOVER_SOURCE_VOTE_ACCOUNT; + ulong floor = coverage_floor( ctx ); + ulong root = ctx->root_slot; + int peer_ok = ctx->peer_tower.valid && ( floor==FD_FAILOVER_SLOT_NULL || ctx->peer_tower.tip>=floor ) && + ( root ==FD_FAILOVER_SLOT_NULL || ctx->peer_tower.tip> root ); + return peer_ok ? FD_FAILOVER_SOURCE_PEER : FD_FAILOVER_SOURCE_VOTE_ACCOUNT; +} + +/* Whether our handoff request is paused at its deadline. */ +static inline int +request_paused( fd_failover_tile_ctx_t const * ctx ) { + return ctx->request.id && !ctx->request.until && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); +} + +/* Run an operator command. If we refuse it we say why. */ +static ulong +apply_control( fd_failover_tile_ctx_t * ctx, + fd_adminctl_failover_req_t const * req, + long now ) { + switch( req->cmd ) { + + case FD_ADMINCTL_FAILOVER_CMD_DEMOTE: { + if( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->request.id && + ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ) { + FD_LOG_NOTICE(( "handoff %lu: operator demote ends the wait for confirmation, giving up the staked identity locally", ctx->request.id )); + request_end( ctx, now, FD_FAILOVER_HANDOFF_DECLINED ); + } + /* Drop the identity without handing it over. The peer is not asked + or told anything, the other machine then needs fenced recovery. */ + if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE ) ) return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; + if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_ACTIVE ) ) return FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; + start_demotion( ctx, 0 ); + return FD_ADMINCTL_RESULT_SUCCESS; + } + + case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: { + int paused = request_paused( ctx ) && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT; + if( FD_UNLIKELY( paused && ctx->request.peer_restarted ) ) { + /* The active restarted before it demoted, so start a new request. */ + FD_LOG_NOTICE(( "handoff %lu was bound to active boot %016lx, which restarted, ending it and starting a new handoff", ctx->request.id, ctx->request.peer.boot_id )); + request_end( ctx, now, FD_FAILOVER_HANDOFF_RESTARTED ); + paused = 0; + } + if( FD_UNLIKELY( paused ) ) { + /* Resume the paused request, a new --address or --port replaces + where we dial. */ + ctx->request.until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); + ctx->request.sent = 0; + ctx->stuck = 0; + if( req->addr ) ctx->cmd_addr = req->addr; + if( req->port ) ctx->cmd_port = req->port; + if( ctx->cmd_addr ) ctx->request.addr = ctx->cmd_addr; + else if( !ctx->request.peer.boot_id && ctx->staked_addr ) ctx->request.addr = ctx->staked_addr; + fd_failover_channel_init_dialer( ctx->channel, ctx->request.addr, dial_port( ctx ) ); + FD_LOG_NOTICE(( "resuming handoff %lu at `" FD_IP4_ADDR_FMT ":%hu` from %s, %s, %s", + ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), dial_port( ctx ), + ctx->cmd_addr ? "`failover promote --address`" : "gossip", + ctx->request.peer.boot_id ? "same authenticated member and boot" : "active has not authenticated yet", + request_wait( ctx ) )); + return FD_ADMINCTL_RESULT_SUCCESS; + } + if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) + return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; + if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_STANDBY ) ) return FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; + uint addr = req->addr ? req->addr : ctx->staked_addr; + if( FD_UNLIKELY( !addr ) ) return FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS; + ctx->cmd_addr = req->addr; + ctx->cmd_port = req->port; + fd_memset( &ctx->request, 0, sizeof(ctx->request) ); + ctx->request.id = ctx->handoff_base+(++ctx->handoff_cnt); + if( FD_UNLIKELY( !ctx->request.id ) ) ctx->request.id = ++ctx->handoff_cnt; + ctx->request.last_id = ctx->request.id; + ctx->request.result = FD_FAILOVER_HANDOFF_PENDING; + ctx->request.addr = addr; + ctx->request.until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); + ctx->last_requested = 1; + ctx->stuck = 0; + ctx->action = FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER; + ctx->session.close_after_send = 0; + fd_failover_channel_init_dialer( ctx->channel, addr, dial_port( ctx ) ); + FD_LOG_NOTICE(( "requesting handoff %lu from the active at `" FD_IP4_ADDR_FMT ":%hu` using %s, authenticating the peer before requesting final state", + ctx->request.id, FD_IP4_ADDR_FMT_ARGS( addr ), dial_port( ctx ), ctx->cmd_addr ? "`failover promote --address`" : "gossip" )); + return FD_ADMINCTL_RESULT_SUCCESS; + } + + case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: { + /* --force takes over without dialing the peer and accepts incomplete + or empty history. */ + int force = !!( req->flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ); + ulong result = promote_guard( ctx, now, force ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) return result; + ulong source = promote_source( ctx ); + /* The operator fenced the peer. Only a request with no local + adoption or key switch in flight can be cancelled for recovery. */ + if( force && ctx->role==FD_FAILOVER_ROLE_STANDBY && ctx->request.id && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) && + ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ) { + FD_LOG_NOTICE(( "handoff %lu cancelled by `failover promote --force` on the operator's word that the peer cannot sign, proceeding with local recovery", ctx->request.id )); + request_end( ctx, now, FD_FAILOVER_HANDOFF_CANCELLED ); + } + if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) { + FD_LOG_WARNING(( "`failover promote --force` stops waiting for the peer's answer to handoff %lu, discarding saved final state because the peer may have voted, coverage floors retained", ctx->handoff.id )); + ctx->peer_tower.valid = 0; + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_CANCELLED ); + } + if( force ) FD_LOG_WARNING(( "operator promotion: --force asserts the peer cannot sign, peer checks are bypassed and incomplete or empty vote history is permitted if needed" )); + if( source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT ) { + fd_failover_tower_t const * saved = &ctx->peer_tower; + if( saved->valid ) { + ulong floor = coverage_floor( ctx ); + if( floor!=FD_FAILOVER_SLOT_NULL && saved->tiptip, floor )); + else if( ctx->root_slot!=FD_FAILOVER_SLOT_NULL && saved->tip<=ctx->root_slot ) + FD_LOG_NOTICE(( "operator promotion: the stored peer tower ends at slot %lu at or below the local root %lu, skipping it", saved->tip, ctx->root_slot )); + } + FD_LOG_NOTICE(( "operator promotion: no eligible saved final tower is available, selecting the vote account" )); + } else { + fd_failover_tower_t const * saved = &ctx->peer_tower; + FD_LOG_NOTICE(( "operator promotion: selecting %s through slot %lu, the tower tile must validate and adopt it before the staked identity is installed", source_name( source ), saved->tip )); + } + start_promotion( ctx, source, NULL, 0UL, force ); + return FD_ADMINCTL_RESULT_SUCCESS; + } + + default: break; + } + return FD_ADMINCTL_RESULT_UNSUPPORTED; +} + +/* The failover status response. */ +static void +status_snapshot( fd_failover_tile_ctx_t const * ctx, + long now, + fd_adminctl_failover_status_resp_t * resp ) { + fd_adminctl_failover_status_resp_init( resp ); + resp->enabled = 1; + resp->role = (uchar)ctx->role; + resp->action = (uchar)ctx->action; + resp->stuck = (uchar)!!ctx->stuck; + resp->link_state = (uchar)fd_failover_channel_state( ctx->channel ); + if( FD_UNLIKELY( fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED ) ) { + resp->peer_role_valid = 1; + resp->peer_role = (uchar)ctx->session.peer_role; + } + resp->peer_boot_id = ctx->session.peer_boot_id; + resp->peer_addr = ctx->request.id ? ctx->request.addr : ctx->staked_addr; + resp->peer_addr_cmd = (uchar)( ctx->request.id && ctx->cmd_addr ); + resp->peer_port = ctx->request.id ? dial_port( ctx ) : ctx->port; + resp->handoff_id = ctx->last_requested ? ctx->request.last_id : ctx->handoff.id; + resp->handoff_result = (uchar)( ctx->last_requested ? ctx->request.result : ctx->handoff.result ); + resp->promote_source = (uchar)promote_source( ctx ); + resp->promote_floor = coverage_floor( ctx ); + resp->promote_result = promote_guard( ctx, now, 0 ); + resp->mode = ctx->hello.mode; + resp->request_paused = (uchar)request_paused( ctx ); +} + +/* Respond to one bus request. The admin tile validated the ABI already. + The response goes out in its own chunk. */ +static void +serve_bus_request( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem, + long now ) { + fd_adminctl_failover_req_t req; + fd_memcpy( &req, ctx->bus_req.payload, sizeof(req) ); + fd_failover_bus_msg_t * out = fd_chunk_to_laddr( ctx->admin_out_mem, ctx->admin_out_chunk ); + fd_memset( out, 0, sizeof(*out) ); + out->nonce = ctx->bus_req.nonce; + + if( FD_UNLIKELY( req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ) ) { + fd_adminctl_failover_status_resp_t status; + status_snapshot( ctx, now, &status ); + out->result = FD_ADMINCTL_RESULT_SUCCESS; + fd_memcpy( out->payload, &status, sizeof(status) ); + } else { + char const * cmd_name = fd_adminctl_failover_cmd_typed( req.cmd ); + FD_LOG_NOTICE(( "`failover %s%s%s` received", cmd_name, + ( req.flags & FD_ADMINCTL_FAILOVER_FLAG_YES ) ? " --yes" : "", + ( req.flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ) ? " --force" : "" )); + ulong result = apply_control( ctx, &req, now ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + char const * hint; + char const * name = control_refusal( result, &hint ); + FD_LOG_WARNING(( "`failover %s` refused with %s, %s", cmd_name, name, hint )); + } + /* An accepted promote or demote says which handoff to follow. */ + ulong handoff_id = 0UL; + if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) { + if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) handoff_id = ctx->request.id; + else if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_DEMOTE ) handoff_id = ctx->handoff.id; + } + fd_adminctl_failover_control_resp_t response = { + .version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, + .role = (uchar)ctx->role, + .action = (uchar)ctx->action, + .handoff_id = handoff_id, + }; + out->result = result; + fd_memcpy( out->payload, &response, sizeof(response) ); + } + + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_RESPONSE, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub ); + ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark ); +} + +/* The active's address comes from gossip, from the newest staked + contact info that is not ours, which is how the active shows up. */ +static void +gossip_commit( fd_failover_tile_ctx_t * ctx, + long now ) { + uint addr = ctx->gossip_socket.addr; + if( FD_UNLIKELY( !addr ) ) return; + if( FD_LIKELY( !fd_memeq( ctx->gossip_origin, ctx->hello.staked_pubkey, 32UL ) ) ) return; + if( FD_UNLIKELY( addr==ctx->own_gossip.addr && ctx->gossip_socket.port==ctx->own_gossip.port ) ) return; + ctx->staked_seen_at = now; + /* Gossip may be the only sign of a peer tenure, so we drop the peer's + tower for good, not only while the contact info is fresh. */ + if( FD_UNLIKELY( ctx->peer_tower.valid ) ) + FD_LOG_NOTICE(( "gossip shows the staked identity at another host, discarding saved final state from before its voting tenure, keeping coverage floors" )); + ctx->peer_tower.valid = 0; + + if( FD_LIKELY( addr==ctx->staked_addr ) ) return; + uint old_addr = ctx->staked_addr; + ctx->staked_addr = addr; + if( FD_LIKELY( !old_addr ) ) { + FD_LOG_NOTICE(( "gossip shows the staked identity at `" FD_IP4_ADDR_FMT "`, a handoff from this machine dials it", FD_IP4_ADDR_FMT_ARGS( addr ) )); + } else { + FD_LOG_NOTICE(( "gossip shows a new address for the staked identity, `" FD_IP4_ADDR_FMT "` before and `" FD_IP4_ADDR_FMT "` now, the next handoff uses the new address", + FD_IP4_ADDR_FMT_ARGS( old_addr ), FD_IP4_ADDR_FMT_ARGS( addr ) )); + } +} + +static inline int +before_frag( fd_failover_tile_ctx_t * ctx, + ulong in_idx, + ulong seq, + ulong sig ) { + if( FD_LIKELY( in_idx==ctx->gossip_in_idx ) ) { + return sig!=FD_GOSSIP_UPDATE_TAG_CONTACT_INFO && sig!=FD_GOSSIP_UPDATE_TAG_CONTACT_INFO_REMOVE; + } + if( FD_LIKELY( in_idx==ctx->tower_in_idx ) ) { + /* A skipped frag may have been a vote. A slot done counts toward the + watermark in after_frag, once the stem has checked the copy. One + abandoned to an overrun must not count, or the final tower would be + taken from the cache with that vote missing. Other frags hold no + vote and count here. */ + if( FD_UNLIKELY( ctx->tower_seen_seq!=ULONG_MAX && seq!=fd_seq_inc( ctx->tower_seen_seq, 1UL ) ) ) ctx->tower_gap = 1; + if( FD_UNLIKELY( sig!=FD_TOWER_SIG_SLOT_DONE ) ) { + ctx->tower_seen_seq = seq; + return 1; + } + return 0; + } + if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { + /* Commands and switch responses. Dropping a switch response here would + leave the switch hanging forever. */ + return sig!=FD_FAILOVER_BUS_REQUEST && sig!=FD_FAILOVER_BUS_SWITCH_RESP; + } + return 0; +} + +static void +during_frag( fd_failover_tile_ctx_t * ctx, + ulong in_idx, + ulong seq FD_PARAM_UNUSED, + ulong sig, + ulong chunk, + ulong sz, + ulong ctl FD_PARAM_UNUSED ) { + if( FD_UNLIKELY( in_idx==ctx->adopt_in_idx ) ) { + if( FD_UNLIKELY( chunkadopt_in_chunk0 || chunk>ctx->adopt_in_wmark || sz!=sizeof(fd_tower_adopt_result_t) ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->adopt_in_chunk0, ctx->adopt_in_wmark )); + } + fd_memcpy( &ctx->adopt_result, fd_chunk_to_laddr_const( ctx->adopt_in_mem, chunk ), sizeof(fd_tower_adopt_result_t) ); + return; + } + if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { + if( FD_UNLIKELY( chunkadmin_in_chunk0 || chunk>ctx->admin_in_wmark || sz!=sizeof(fd_failover_bus_msg_t) ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->admin_in_chunk0, ctx->admin_in_wmark )); + } + fd_failover_bus_msg_t const * msg = fd_chunk_to_laddr_const( ctx->admin_in_mem, chunk ); + if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_SWITCH_RESP ) ) { + fd_memcpy( &ctx->id_switch.response, msg->payload, sizeof(ctx->id_switch.response) ); + ctx->id_switch.response_nonce = msg->nonce; + return; + } + fd_memcpy( &ctx->bus_req, msg, sizeof(fd_failover_bus_msg_t) ); + return; + } + if( FD_UNLIKELY( in_idx==ctx->tower_in_idx ) ) { + if( FD_UNLIKELY( chunktower_in_chunk0 || chunk>ctx->tower_in_wmark || sz!=sizeof(fd_tower_msg_t) ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->tower_in_chunk0, ctx->tower_in_wmark )); + } + fd_memcpy( &ctx->slot_done, fd_chunk_to_laddr_const( ctx->tower_in_mem, chunk ), sizeof(fd_tower_slot_done_t) ); + return; + } + if( FD_UNLIKELY( in_idx!=ctx->gossip_in_idx ) ) return; + if( FD_UNLIKELY( chunkgossip_in_chunk0 || chunk>ctx->gossip_in_wmark || sz>ctx->gossip_in_mtu ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->gossip_in_chunk0, ctx->gossip_in_wmark )); + } + /* The link is unreliable, so we only copy here and after_frag commits + once the stem has checked the copy. A remove leaves the socket + zero. */ + fd_gossip_update_message_t const * msg = fd_chunk_to_laddr_const( ctx->gossip_in_mem, chunk ); + fd_memcpy( ctx->gossip_origin, msg->origin, 32UL ); + ctx->gossip_socket.addr = 0U; + ctx->gossip_socket.port = (ushort)0; + if( FD_LIKELY( sig==FD_GOSSIP_UPDATE_TAG_CONTACT_INFO ) ) { + fd_gossip_socket_t const * socket = &msg->contact_info->value->sockets[ FD_GOSSIP_CONTACT_INFO_SOCKET_GOSSIP ]; + ctx->gossip_socket.addr = fd_uint_if ( !socket->is_ipv6, socket->ip4, 0U ); + ctx->gossip_socket.port = fd_ushort_if( !socket->is_ipv6, socket->port, (ushort)0 ); + } +} + +static void +after_frag( fd_failover_tile_ctx_t * ctx, + ulong in_idx, + ulong seq, + ulong sig, + ulong sz FD_PARAM_UNUSED, + ulong tsorig FD_PARAM_UNUSED, + ulong tspub FD_PARAM_UNUSED, + fd_stem_context_t * stem FD_PARAM_UNUSED ) { + if( FD_UNLIKELY( in_idx==ctx->adopt_in_idx ) ) { + ctx->adopt_result_id = sig; + ctx->adopt_result_fresh = 1; + return; + } + if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { + if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_SWITCH_RESP ) ) { + /* The drain still reads the last switch result, so we copy a + response into it only after the stem and nonce checks pass. */ + if( FD_LIKELY( switch_answer( ctx, ctx->id_switch.response_nonce ) ) ) ctx->id_switch.result = ctx->id_switch.response; + return; + } + /* Send the response from after_credit, where a publish credit is available. */ + ctx->bus_req_fresh = 1; + return; + } + if( FD_UNLIKELY( in_idx==ctx->tower_in_idx ) ) { + ctx->slot_done_fresh = 1; + ctx->tower_seen_seq = seq; + return; + } + if( FD_UNLIKELY( in_idx!=ctx->gossip_in_idx ) ) return; + gossip_commit( ctx, fd_failover_clock() ); +} + +static inline void +after_credit( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem, + int * opt_poll_in FD_PARAM_UNUSED, + int * charge_busy ) { + if( FD_UNLIKELY( !ctx->member_cert_set ) ) request_member_cert( ctx ); + if( FD_UNLIKELY( ctx->slot_done_fresh ) ) { + ctx->slot_done_fresh = 0; + consume_slot_done( ctx, &ctx->slot_done ); + } + step_controller( ctx, stem ); + + long now = fd_failover_clock(); + if( FD_LIKELY( now>=ctx->poll_at ) ) { + int busy = 0; + peer_poll( ctx, now, &busy ); + ctx->poll_at = busy ? now : now+FD_FAILOVER_TILE_POLL_NANOS; + if( busy ) *charge_busy = 1; + } + pending_flush( ctx, now ); + if( FD_UNLIKELY( ctx->bus_req_fresh ) ) { + ctx->bus_req_fresh = 0; + serve_bus_request( ctx, stem, now ); + *charge_busy = 1; + } + + if( FD_UNLIKELY( ctx->stuck!=ctx->stuck_logged ) ) { + ctx->stuck_logged = ctx->stuck; + if( !ctx->stuck ) FD_LOG_NOTICE(( "failover is no longer stuck, %s", ctx->role==FD_FAILOVER_ROLE_ACTIVE ? "this machine is active" : "this machine is standby" )); + } +} + +static ulong +populate_allowed_seccomp( fd_topo_t const * topo, + fd_topo_tile_t const * tile, + ulong out_cnt, + struct sock_filter * out ) { + fd_failover_tile_ctx_t const * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id ); + populate_sock_filter_policy_fd_failover_tile( out_cnt, out, (uint)fd_log_private_logfile_fd(), (uint)fd_failover_channel_listen_fd( ctx->channel ) ); + return sock_filter_policy_fd_failover_tile_instr_cnt; +} + +static ulong +rlimit_file_cnt( fd_topo_t const * topo FD_PARAM_UNUSED, + fd_topo_tile_t const * tile FD_PARAM_UNUSED ) { + return FD_FAILOVER_CHANNEL_CANDIDATE_MAX + 1UL + 8UL; +} + +static ulong +populate_allowed_fds( fd_topo_t const * topo, + fd_topo_tile_t const * tile, + ulong out_fds_cnt, + int * out_fds ) { + fd_failover_tile_ctx_t const * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id ); + int logfile_fd = fd_log_private_logfile_fd(); + int listen_fd = fd_failover_channel_listen_fd( ctx->channel ); + + ulong required_fds = 1UL + (ulong)(-1!=logfile_fd) + (ulong)(-1!=listen_fd); + if( FD_UNLIKELY( out_fds_cnt +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__i386__) +# define ARCH_NR AUDIT_ARCH_I386 +#elif defined(__x86_64__) +# define ARCH_NR AUDIT_ARCH_X86_64 +#elif defined(__aarch64__) +# define ARCH_NR AUDIT_ARCH_AARCH64 +#elif defined(__riscv) && __riscv_xlen == 64 +# define ARCH_NR AUDIT_ARCH_RISCV64 +#else +# error "Target architecture is unsupported by seccomp." +#endif + +#define FD_SECCOMP_ARG_LO_OFFSET(argno) ( offsetof( struct seccomp_data, args[(argno)] ) ) +#define FD_SECCOMP_ARG_HI_OFFSET(argno) ( offsetof( struct seccomp_data, args[(argno)] ) + 4U ) + +#define FD_SECCOMP_ARG_LO(x) ((uint)(((ulong)(uint)(int)(x) ) & 0xffffffffUL)) +#define FD_SECCOMP_ARG_HI(x) ((uint)(((ulong)(x) >> 32) & 0xffffffffUL)) + +static const uint sock_filter_policy_fd_failover_tile_instr_cnt = 46; + +static void populate_sock_filter_policy_fd_failover_tile( ulong out_cnt, struct sock_filter out[ static 46 ], uint logfile_fd, uint listen_fd ) { + FD_TEST( out_cnt >= 46 ); + struct sock_filter filter[46] = { + /* validate architecture */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, arch ) )), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ARCH_NR, 0, /* RET_KILL_PROCESS */ 10 ), + /* load syscall number */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, ( offsetof( struct seccomp_data, nr ) )), + /* check write */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_write, /* check_write */ 10, 0 ), + /* check fsync */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_fsync, /* check_fsync */ 15, 0 ), + /* allow getrandom */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_getrandom, /* RET_ALLOW */ 7, 0 ), + /* check socket */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_socket, /* check_socket */ 17, 0 ), + /* allow connect */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_connect, /* RET_ALLOW */ 5, 0 ), + /* check accept4 */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_accept4, /* check_accept4 */ 23, 0 ), + /* allow recvfrom */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_recvfrom, /* RET_ALLOW */ 3, 0 ), + /* allow sendto */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_sendto, /* RET_ALLOW */ 2, 0 ), + /* check close */ + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, SYS_close, /* check_close */ 26, 0 ), +// RET_KILL_PROCESS: + /* default deny */ + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// RET_ALLOW: + /* allow */ + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_write: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 0x00000002U, /* write_ALLOW */ 3, /* or_1 */ 0 ), +// or_1: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(logfile_fd)), /* write_ALLOW */ 1, /* write_KILL */ 0 ), +// write_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// write_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_fsync: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(logfile_fd)), /* fsync_ALLOW */ 1, /* fsync_KILL */ 0 ), +// fsync_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// fsync_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_socket: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(AF_INET), /* and_2 */ 0, /* socket_KILL */ 4 ), +// and_2: + /* arg 1 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(1)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC), /* and_3 */ 0, /* socket_KILL */ 2 ), +// and_3: + /* arg 2 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(2)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 0x00000000U, /* socket_ALLOW */ 1, /* socket_KILL */ 0 ), +// socket_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// socket_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_accept4: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(listen_fd)), /* and_4 */ 0, /* accept4_KILL */ 2 ), +// and_4: + /* arg 3 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(3)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, FD_SECCOMP_ARG_LO(SOCK_NONBLOCK|SOCK_CLOEXEC), /* accept4_ALLOW */ 1, /* accept4_KILL */ 0 ), +// accept4_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// accept4_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), +// check_close: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, 0x00000002U, /* close_KILL */ 4, /* or_5 */ 0 ), +// or_5: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(logfile_fd)), /* close_KILL */ 2, /* or_6 */ 0 ), +// or_6: + /* arg 0 low 32 bits */ + BPF_STMT( BPF_LD | BPF_W | BPF_ABS, FD_SECCOMP_ARG_LO_OFFSET(0)), + BPF_JUMP( BPF_JMP | BPF_JEQ | BPF_K, ((uint)(listen_fd)), /* close_KILL */ 0, /* close_ALLOW */ 1 ), +// close_KILL: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS ), +// close_ALLOW: + BPF_STMT( BPF_RET | BPF_K, SECCOMP_RET_ALLOW ), + }; + fd_memcpy( out, filter, sizeof( filter ) ); +} + +#endif /* defined(__linux__) */ + +#endif /* HEADER_fd_src_discof_failover_generated_fd_failover_tile_seccomp_h */ diff --git a/src/discof/failover/test_failover_tile.c b/src/discof/failover/test_failover_tile.c new file mode 100644 index 00000000000..f3787fad524 --- /dev/null +++ b/src/discof/failover/test_failover_tile.c @@ -0,0 +1,528 @@ +#include "fd_failover_channel.c" +#include "fd_failover_tile.c" + +#include + +#define OWN_GOSSIP_ADDR FD_IP4_ADDR(10,0,0,1) +#define OWN_GOSSIP_PORT ((ushort)8001) + +static fd_frag_meta_t pub_mcache[ 8 ]; +static ulong pub_cr_avail; +static ulong pub_min_cr_avail; +static int pub_reliable; +static fd_stem_context_t stem[ 1 ]; +static uchar bus_mem[ 4096 ] __attribute__((aligned(128))); + +static void +stem_init( void ) { + static fd_frag_meta_t * mcaches[ 1 ]; + static ulong seqs[ 1 ]; + static ulong depths[ 1 ]; + mcaches[ 0 ] = pub_mcache; + seqs[ 0 ] = 0UL; + depths[ 0 ] = 8UL; + pub_cr_avail = 64UL; + pub_min_cr_avail = 64UL; + pub_reliable = 0; + *stem = (fd_stem_context_t){ + .mcaches = mcaches, .seqs = seqs, .depths = depths, + .cr_avail = &pub_cr_avail, .min_cr_avail = &pub_min_cr_avail, + .cr_decrement_amount = 1UL, .out_reliable = &pub_reliable, + }; +} + +static uchar ch_mem[ 8UL<<20 ] __attribute__((aligned(128))); +static fd_failover_tile_ctx_t ctl[ 1 ]; + +#define OUR_BOOT_ID (1000UL) + +static fd_failover_tile_ctx_t * +controller_init( ulong role ) { + stem_init(); + fd_failover_tile_ctx_t * ctx = ctl; + fd_memset( ctx, 0, sizeof(fd_failover_tile_ctx_t) ); + ctx->role = role; + ctx->hello.role = (uchar)role; + ctx->hello.boot_id = OUR_BOOT_ID; + fd_memset( ctx->hello.junk_pubkey, 0x11, 32UL ); + fd_memset( ctx->hello.staked_pubkey, 0x5A, 32UL ); + ctx->replay_slot = 100UL; + ctx->root_slot = 90UL; + ctx->last_vote_slot = 99UL; + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT; + ctx->deadline_slot = FD_FAILOVER_SLOT_NULL; + ctx->peer_floor = FD_FAILOVER_SLOT_NULL; + ctx->own_floor = FD_FAILOVER_SLOT_NULL; + ctx->tower_seen_seq = ULONG_MAX; + ctx->handoff_base = 5000UL; + ctx->gossip_in_idx = ULONG_MAX; + ctx->tower_in_idx = ULONG_MAX; + ctx->admin_in_idx = ULONG_MAX; + ctx->adopt_in_idx = ULONG_MAX; + ctx->admin_out_idx = 0UL; + ctx->admin_out_mem = (fd_wksp_t *)bus_mem; + ctx->adopt_out_idx = 0UL; + ctx->adopt_out_mem = (fd_wksp_t *)bus_mem; + ctx->own_gossip.addr = OWN_GOSSIP_ADDR; + ctx->own_gossip.port = fd_ushort_bswap( OWN_GOSSIP_PORT ); + ctx->channel = fd_failover_channel_join( fd_failover_channel_new( ch_mem ) ); + FD_TEST( ctx->channel ); + ctx->channel->self_hello = ctx->hello; + ctx->session.state = fd_failover_channel_state( ctx->channel ); + return ctx; +} + +static void +controller_fini( fd_failover_tile_ctx_t * ctx ) { + fd_failover_channel_fini( ctx->channel ); +} + +static void +pair( fd_failover_tile_ctx_t * ctx, + ulong boot_id, + ulong peer_role, + long now ) { + ctx->channel->state = FD_FAILOVER_SESSION_PAIRED; + ctx->channel->peer_hello.boot_id = boot_id; + ctx->channel->peer_hello.role = (uchar)peer_role; + sync_session( ctx ); + (void)now; + fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x22, 32UL ); +} + +static void +make_tower( fd_failover_tower_t * tower, + ulong tip ) { + fd_compact_tower_sync_serde_t serde; + fd_memset( &serde, 0, sizeof(serde) ); + serde.root = tip-2UL; + serde.lockouts_cnt = 1; + serde.lockouts[ 0 ].offset = 2UL; + serde.lockouts[ 0 ].confirmation_count = 1; + FD_TEST( !fd_compact_tower_sync_ser( &serde, tower->state, FD_FAILOVER_TOWER_STATE_MAX, &tower->sz ) ); + tower->valid = 1; + tower->tip = tip; +} + +static void +deliver( fd_failover_tile_ctx_t * ctx, + ushort type, + void const * payload, + ulong payload_sz ) { + fd_memcpy( ctx->rx, payload, payload_sz ); + handle_control( ctx, type, payload_sz, 1000L ); +} + +static void +deliver_ack( fd_failover_tile_ctx_t * ctx, + ulong handoff_id ) { + uchar payload[ sizeof(fd_failover_promote_ack_t) ]; + deliver( ctx, (ushort)FD_FAILOVER_MSG_PROMOTE_ACK, payload, fd_failover_promote_ack_encode( payload, handoff_id ) ); +} + +static void +switch_ok( fd_failover_tile_ctx_t * ctx, + ulong watermark ) { + ctx->id_switch.result.result = FD_FAILOVER_SWITCH_OK; + ctx->id_switch.result.tower_watermark = watermark; + FD_TEST( switch_answer( ctx, ctx->id_switch.request_id ) ); +} + +static fd_failover_demoted_t +pending_demoted( fd_failover_tile_ctx_t const * ctx ) { + FD_TEST( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ); + fd_failover_demoted_t demoted; + FD_TEST( fd_failover_demoted_decode( &demoted, ctx->tx.payload, ctx->tx.sz ) ); + return demoted; +} + +static void +test_handoff( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + make_tower( &ctx->current_tower, 99UL ); + fd_failover_handoff_request_t req = { .handoff_id=42UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH ); + step_controller( ctx, stem ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK && !ctx->tx.valid ); + switch_ok( ctx, 0UL ); + step_controller( ctx, stem ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY ); + FD_TEST( pending_demoted( ctx ).handoff_id==42UL ); + ctx->tx.valid = 0; + ctx->handoff.delivery = FD_FAILOVER_DELIVERY_SENT; + deliver_ack( ctx, 42UL ); + FD_TEST( ctx->taken && ctx->handoff.result==FD_FAILOVER_HANDOFF_TAKEN ); + controller_fini( ctx ); +} + +static void +test_no_final_state( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_handoff_request_t req = { .handoff_id=43UL, .target_boot_id=OUR_BOOT_ID }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + fd_failover_handoff_result_t result; + FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); + FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); + FD_TEST( result.handoff_id==43UL && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.request_id && !ctx->promotion.adopt_id ); + controller_fini( ctx ); +} + +/* test_request_replay_behind: a standby more than 64 slots behind our + last vote, or one with no replay slot yet, gets REPLAY_BEHIND and we + keep the identity, one exactly 64 behind is served. */ +static void +test_request_replay_behind( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + ctx->last_vote_slot = 200UL; + make_tower( &ctx->current_tower, 200UL ); + ulong behind[ 2 ] = { 200UL-FD_FAILOVER_DEADLINE_SLOTS-1UL, FD_FAILOVER_SLOT_NULL }; + for( ulong i=0UL; i<2UL; i++ ) { + fd_failover_handoff_request_t req = { .handoff_id=49UL+i, .target_boot_id=OUR_BOOT_ID, .replay_slot=behind[ i ] }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + fd_failover_handoff_result_t result; + FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); + FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); + FD_TEST( result.handoff_id==49UL+i && result.result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->handoff.id ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.request_id ); + ctx->tx.valid = 0; + ctx->session.close_after_send = 0; + } + fd_failover_handoff_request_t req = { .handoff_id=51UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=200UL-FD_FAILOVER_DEADLINE_SLOTS }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->handoff.id==51UL ); + controller_fini( ctx ); +} + +/* test_demote_drain: after the junk switch the drain is its own action, + status reports it, and an ACK or the same request during it is ignored. */ +static void +test_demote_drain( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + make_tower( &ctx->current_tower, 99UL ); + fd_failover_handoff_request_t req = { .handoff_id=44UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + step_controller( ctx, stem ); + ctx->tower_seen_seq = 796UL; + switch_ok( ctx, 800UL ); + step_controller( ctx, stem ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN ); + fd_adminctl_failover_status_resp_t status; + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.action==(uchar)FD_FAILOVER_ACTION_DEMOTE_DRAIN ); + deliver_ack( ctx, 44UL ); + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN && !ctx->taken && !ctx->tx.valid ); + ctx->tower_seen_seq = 799UL; + step_controller( ctx, stem ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && pending_demoted( ctx ).handoff_id==44UL ); + controller_fini( ctx ); +} + +/* A request of ours bound to boot 77 of the member whose junk key is + 0x22, as the tests' pair() makes it. */ +static void +open_request( fd_failover_tile_ctx_t * ctx, + ulong action ) { + ctx->request.id = 900UL; + ctx->request.last_id = 900UL; + ctx->request.result = FD_FAILOVER_HANDOFF_PENDING; + ctx->request.addr = FD_IP4_ADDR(10,0,0,2); + ctx->request.until = fd_long_sat_add( 1000L, FD_FAILOVER_CHANNEL_IDLE_NANOS ); + ctx->request.peer.boot_id = 77UL; + fd_memset( ctx->request.peer.junk, 0x22, 32UL ); + ctx->last_requested = 1; + ctx->action = action; +} + +/* test_result_wait_peer_restart: the old active restarts before it + confirms our ACK. Its new boot cannot, so the wait ends and we stay + active. A standby waiting to have its refusal confirmed ends the + same way and stays standby. */ +static void +test_result_wait_peer_restart( void ) { + for( ulong active=0UL; active<2UL; active++ ) { + fd_failover_tile_ctx_t * ctx = controller_init( active ? FD_FAILOVER_ROLE_ACTIVE : FD_FAILOVER_ROLE_STANDBY ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + pair( ctx, 78UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + int busy = 0; + peer_poll( ctx, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->request.id ); + FD_TEST( ctx->request.result==( active ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ) ); + FD_TEST( ctx->role==( active ? FD_FAILOVER_ROLE_ACTIVE : FD_FAILOVER_ROLE_STANDBY ) ); + FD_TEST( ctx->stuck==!active ); + controller_fini( ctx ); + } + + /* While we wait for final state a restart still pauses the request. */ + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ); + pair( ctx, 78UL, FD_FAILOVER_ROLE_ACTIVE, 1000L ); + int busy = 0; + peer_poll( ctx, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && ctx->request.id && !ctx->request.until && ctx->stuck ); + controller_fini( ctx ); +} + +/* test_session_change_drops_result: a queued HANDOFF_RESULT belongs to + its session, a new one drops it and the requester asks again. A + queued DEMOTED stays for the handoff target. */ +static void +test_session_change_drops_result( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_handoff_request_t req = { .handoff_id=46UL, .target_boot_id=OUR_BOOT_ID+1UL, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT && ctx->session.close_after_send ); + ctx->channel->generation++; + pair( ctx, 79UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + FD_TEST( !ctx->tx.valid && !ctx->session.close_after_send ); + controller_fini( ctx ); + + ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + make_tower( &ctx->current_tower, 99UL ); + req.target_boot_id = OUR_BOOT_ID; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + step_controller( ctx, stem ); + switch_ok( ctx, 0UL ); + step_controller( ctx, stem ); + FD_TEST( pending_demoted( ctx ).handoff_id==46UL ); + ctx->channel->generation++; + pair( ctx, 80UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + FD_TEST( pending_demoted( ctx ).handoff_id==46UL ); + controller_fini( ctx ); +} + +static ulong +operator_cmd( fd_failover_tile_ctx_t * ctx, + ulong cmd, + uint addr, + ushort port ) { + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=cmd, .addr=addr, .port=port }; + return apply_control( ctx, &req, 1000L ); +} + +/* test_request_address: status shows where the open request dials, a + gossip address fixed when it opened, and after it the live one. A + resume replaces the address or port it gives and keeps the other, one + that gives neither keeps both, and a bound request takes a given + address too. */ +static void +test_request_address( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + ctx->port = 8010; + ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + ctx->staked_addr = FD_IP4_ADDR(10,0,0,3); + fd_adminctl_failover_status_resp_t status; + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.peer_addr==FD_IP4_ADDR(10,0,0,2) && !status.peer_addr_cmd && status.peer_port==8010 ); + + request_pause( ctx, 1000L, "test" ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, FD_IP4_ADDR(10,0,0,4), 9001 )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,4) && dial_port( ctx )==9001 ); + request_pause( ctx, 1000L, "test" ); + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.request_paused && status.mode==(uchar)FD_FAILOVER_MODE_TOWER ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, FD_IP4_ADDR(10,0,0,5), 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,5) && dial_port( ctx )==9001 ); + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.peer_addr==FD_IP4_ADDR(10,0,0,5) && status.peer_addr_cmd && status.peer_port==9001 && !status.request_paused ); + request_pause( ctx, 1000L, "test" ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 9002 )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,5) && dial_port( ctx )==9002 ); + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.peer_addr_cmd ); + request_pause( ctx, 1000L, "test" ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,5) && dial_port( ctx )==9002 ); + ctx->request.peer.boot_id = 77UL; + request_pause( ctx, 1000L, "test" ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, FD_IP4_ADDR(10,0,0,6), 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,6) && ctx->request.peer.boot_id==77UL ); + + request_end( ctx, 1000L, FD_FAILOVER_HANDOFF_DECLINED ); + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.peer_addr==FD_IP4_ADDR(10,0,0,3) && !status.peer_addr_cmd && status.peer_port==8010 ); + controller_fini( ctx ); +} + +/* test_abort_answers_requester: a handoff demotion whose junk switch + fails keeps the identity and tells the requester at once. */ +static void +test_abort_answers_requester( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + make_tower( &ctx->current_tower, 99UL ); + fd_failover_handoff_request_t req = { .handoff_id=47UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + step_controller( ctx, stem ); + ctx->id_switch.result.result = FD_FAILOVER_SWITCH_ERR_DISABLED; + FD_TEST( switch_answer( ctx, ctx->id_switch.request_id ) ); + step_controller( ctx, stem ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE && ctx->stuck ); + fd_failover_handoff_result_t result; + FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT && ctx->tx.to.boot_id==77UL && ctx->session.close_after_send ); + FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); + FD_TEST( result.handoff_id==47UL && result.result==FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + controller_fini( ctx ); +} + +/* test_wait_result_redials: a request that paused while its promotion + ran dials again once the promotion ends, so the bound peer can confirm + and a restarted one ends the request. */ +static void +test_wait_result_redials( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + open_request( ctx, FD_FAILOVER_ACTION_PROMOTE_SWITCH ); + ctx->request.until = 0L; + ctx->promotion.from_peer = 1; + ctx->promotion.peer = ctx->request.peer; + ctx->promotion.handoff_id = 900UL; + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_STAKED; + ctx->id_switch.request_id = 1UL; + switch_ok( ctx, 0UL ); + step_controller( ctx, stem ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + FD_TEST( ctx->request.until && ctx->channel->peer_addr==FD_IP4_ADDR(10,0,0,2) ); + controller_fini( ctx ); +} + +/* test_refused_command_keeps_stuck: only an accepted command clears + stuck. */ +static void +test_refused_command_keeps_stuck( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + ctx->stuck = 1; + ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_DEMOTE, 0U, 0 )==FD_FAILOVER_CONTROL_RESULT_BAD_ROLE && ctx->stuck ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS && !ctx->stuck ); + controller_fini( ctx ); +} + +/* test_dialed_standby: a request that pairs with a standby ends as its + own outcome, not as declined. */ +static void +test_dialed_standby( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + int busy = 0; + peer_poll( ctx, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->request.id && ctx->request.result==FD_FAILOVER_HANDOFF_NOT_ACTIVE ); + controller_fini( ctx ); +} + +/* test_restarted_peer_new_request: once the active boot a request is + bound to restarted, `failover promote` starts a new request instead of + resuming one that can never finish, and the accepted command answers + with its id. */ +static void +test_restarted_peer_new_request( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ); + pair( ctx, 78UL, FD_FAILOVER_ROLE_ACTIVE, 1000L ); + int busy = 0; + peer_poll( ctx, 1000L, &busy ); + FD_TEST( ctx->request.id==900UL && ctx->request.peer_restarted && !ctx->request.until ); + + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_HANDOFF }; + ctx->bus_req.nonce = 7UL; + fd_memcpy( ctx->bus_req.payload, &req, sizeof(req) ); + serve_bus_request( ctx, stem, 1000L ); + FD_TEST( ctx->request.id && ctx->request.id!=900UL && !ctx->request.peer.boot_id && !ctx->request.peer_restarted ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ); + fd_failover_bus_msg_t const * out = (fd_failover_bus_msg_t const *)bus_mem; + fd_adminctl_failover_control_resp_t response; + fd_memcpy( &response, out->payload, sizeof(response) ); + FD_TEST( out->nonce==7UL && out->result==FD_ADMINCTL_RESULT_SUCCESS && response.handoff_id==ctx->request.id ); + controller_fini( ctx ); +} + +/* test_dial_backoff_reset: a new dial target starts from the shortest + backoff, whatever the last target reached. */ +static void +test_dial_backoff_reset( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + ctx->channel->backoff = ctx->channel->backoff_max; + fd_failover_channel_init_dialer( ctx->channel, FD_IP4_ADDR(10,0,0,2), 8010 ); + FD_TEST( ctx->channel->backoff==ctx->channel->backoff_min ); + controller_fini( ctx ); +} + +/* test_listen_slot: a full table makes room for a newcomer the listener + admitted. The address with the most candidates loses its oldest, + else the oldest of all goes. An authenticated, dialed or paired + candidate is never closed. */ +static void +fill( fd_failover_channel_t * ch, + ulong idx, + uint address, + long deadline ) { + struct candidate * c = &ch->candidates[ idx ]; + if( c->fd!=-1 ) close( c->fd ); + fd_memset( c, 0, sizeof(*c) ); + c->fd = open( "/dev/null", O_RDONLY|O_CLOEXEC ); + FD_TEST( c->fd!=-1 ); + c->tls.fd = -1; + c->phase = PHASE_TLS; + c->address = address; + c->deadline = deadline; +} + +static void +test_listen_slot( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + fd_failover_channel_t * ch = ctx->channel; + FD_TEST( listen_slot( ch, 1000L )==0UL ); + for( ulong i=0UL; icandidates[ 3 ].fd==-1 ); + fill( ch, 3UL, FD_IP4_ADDR(10,2,0,1), 200L ); + FD_TEST( listen_slot( ch, 1000L )==0UL ); + fill( ch, 0UL, FD_IP4_ADDR(10,2,0,2), 201L ); + ch->candidates[ 1 ].phase = PHASE_READY; + ch->candidates[ 2 ].dialed = 1; + ch->paired_idx = 4; + FD_TEST( listen_slot( ch, 1000L )==5UL ); + ch->paired_idx = -1; + for( ulong i=0UL; icandidates[ i ].fd!=-1 ) { + close( ch->candidates[ i ].fd ); + ch->candidates[ i ].fd = -1; + } + controller_fini( ctx ); +} + +int +main( int argc, char ** argv ) { + fd_boot( &argc, &argv ); + FD_TEST( fd_failover_channel_footprint()<=sizeof(ch_mem) ); + test_handoff(); + test_no_final_state(); + test_request_replay_behind(); + test_demote_drain(); + test_result_wait_peer_restart(); + test_session_change_drops_result(); + test_request_address(); + test_abort_answers_requester(); + test_wait_result_redials(); + test_refused_command_keeps_stuck(); + test_dialed_standby(); + test_restarted_peer_new_request(); + test_dial_backoff_reset(); + test_listen_slot(); + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} From 9b08594b3393c2716afe68e6443ee3284f7508fb Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Thu, 8 Oct 2026 23:02:33 +0000 Subject: [PATCH 10/23] failover: add the promote, demote and status commands --- src/app/firedancer-dev/main.h | 2 + src/app/firedancer/Local.mk | 1 + src/app/firedancer/commands/failover.c | 426 +++++++++++++++++++++++++ src/app/firedancer/main.c | 2 + src/app/shared/fd_action.h | 9 + 5 files changed, 440 insertions(+) create mode 100644 src/app/firedancer/commands/failover.c diff --git a/src/app/firedancer-dev/main.h b/src/app/firedancer-dev/main.h index 91e109eee07..e0ab2d3670f 100644 --- a/src/app/firedancer-dev/main.h +++ b/src/app/firedancer-dev/main.h @@ -205,6 +205,7 @@ extern action_t fd_action_mem; extern action_t fd_action_netconf; extern action_t fd_action_set_identity; extern action_t fd_action_get_identity; +extern action_t fd_action_failover; extern action_t fd_action_ps; extern action_t fd_action_version; extern action_t fd_action_bench; @@ -246,6 +247,7 @@ action_t * ACTIONS[] = { &fd_action_netconf, &fd_action_set_identity, &fd_action_get_identity, + &fd_action_failover, &fd_action_ps, &fd_action_help, &fd_action_metrics, diff --git a/src/app/firedancer/Local.mk b/src/app/firedancer/Local.mk index 26896862d3b..91fd45f0415 100644 --- a/src/app/firedancer/Local.mk +++ b/src/app/firedancer/Local.mk @@ -24,6 +24,7 @@ $(call add-objs,commands/shred_version,fd_firedancer) $(call add-objs,commands/snapshot_create,fd_firedancer) $(call add-objs,commands/set_identity,fd_firedancer) $(call add-objs,commands/get_identity,fd_firedancer) +$(call add-objs,commands/failover,fd_firedancer) $(call add-objs,commands/adminctl_client,fd_firedancer) $(call add-objs,commands/monitor_gossip/monitor_gossip commands/monitor_gossip/gossip_diag,fd_firedancer) $(call add-objs,commands/wait,fd_firedancer) diff --git a/src/app/firedancer/commands/failover.c b/src/app/firedancer/commands/failover.c new file mode 100644 index 00000000000..5ae7bd39e6f --- /dev/null +++ b/src/app/firedancer/commands/failover.c @@ -0,0 +1,426 @@ +#define _GNU_SOURCE +#include "adminctl_client.h" +#include "../../shared/fd_config.h" +#include "../../shared/fd_action.h" +#include "../../../discof/failover/fd_failover_proto.h" +#include "../../../disco/topo/fd_dns_resolve.h" +#include "../../../util/net/fd_ip4.h" + +#include +#include +#include +#include + +static char const * const ACTION_NAMES[] = { + "idle", "demote, switching to the junk key", "demote, waiting for the peer's answer", + "promote, waiting for replay", "promote, waiting for vote history adoption", "promote, switching to the staked key", + "handoff, requesting the active's final vote state", "handoff, waiting for completion", + "demote, waiting for the tower to drain" +}; +static char const * const SESSION_NAMES[] = { + "listening", "dialing", "hello", "paired", "backoff" +}; +static char const * const SOURCE_NAMES[] = { + "the stored peer tower", "the vote account (automatic fallback)" +}; +static char const * const HANDOFF_NAMES[] = { + "nothing handed over", "pending", "taken", "declined", "restarted", "cancelled by `failover promote --force`", + "the dialed machine was a standby" +}; + +#define ACTION_NAME_CNT ( sizeof(ACTION_NAMES )/sizeof(ACTION_NAMES [ 0 ]) ) +#define SESSION_NAME_CNT ( sizeof(SESSION_NAMES)/sizeof(SESSION_NAMES[ 0 ]) ) +#define SOURCE_NAME_CNT ( sizeof(SOURCE_NAMES )/sizeof(SOURCE_NAMES [ 0 ]) ) +#define HANDOFF_NAME_CNT ( sizeof(HANDOFF_NAMES)/sizeof(HANDOFF_NAMES[ 0 ]) ) + +FD_STATIC_ASSERT( ACTION_NAME_CNT ==FD_FAILOVER_ACTION_CNT, action_names ); +FD_STATIC_ASSERT( SESSION_NAME_CNT==FD_FAILOVER_SESSION_CNT, session_names ); +FD_STATIC_ASSERT( SOURCE_NAME_CNT ==FD_FAILOVER_SOURCE_CNT, source_names ); +FD_STATIC_ASSERT( HANDOFF_NAME_CNT==FD_FAILOVER_HANDOFF_CNT, handoff_names ); + +/* Strips key and its value. fd_env_strip_cmdline_cstr silently drops a + key given last with no value and takes a flag that follows the key as + its value, so we refuse both before we strip. */ +static char const * +strip_value( int * pargc, + char *** pargv, + char const * key ) { + for( int i=0; i<*pargc; i++ ) { + if( FD_UNLIKELY( !strcmp( (*pargv)[ i ], key ) && ( i+1==*pargc || !strncmp( (*pargv)[ i+1 ], "--", 2UL ) ) ) ) { + FD_LOG_ERR(( "`%s` needs a value", key )); + } + } + return fd_env_strip_cmdline_cstr( pargc, pargv, key, NULL, NULL ); +} + +static void +failover_cmd_args( int * pargc, + char *** pargv, + args_t * args ) { + char const * name = strip_value( pargc, pargv, "--name" ); + char const * address = strip_value( pargc, pargv, "--address" ); + char const * port = strip_value( pargc, pargv, "--port" ); + if( FD_UNLIKELY( name ) ) fd_cstr_ncpy( args->failover.name, name, sizeof(args->failover.name) ); + args->failover.yes = fd_env_strip_cmdline_contains( pargc, pargv, "--yes" ); + args->failover.force = fd_env_strip_cmdline_contains( pargc, pargv, "--force" ); + + if( FD_UNLIKELY( !( *pargc ) ) ) { + FD_LOG_ERR(( "missing subcommand, supported: status, promote, demote" )); + } + /* `promote --force` bypasses checks and takes the identity immediately + without a clean hand-off. */ + char const * cmd = **pargv; + if( !strcmp( cmd, "status" ) ) args->failover.cmd = (int)FD_ADMINCTL_FAILOVER_CMD_STATUS; + else if( !strcmp( cmd, "demote" ) ) args->failover.cmd = (int)FD_ADMINCTL_FAILOVER_CMD_DEMOTE; + else if( !strcmp( cmd, "promote" ) ) args->failover.cmd = (int)( args->failover.force ? FD_ADMINCTL_FAILOVER_CMD_PROMOTE : FD_ADMINCTL_FAILOVER_CMD_HANDOFF ); + else FD_LOG_ERR(( "unknown subcommand `%s`, supported: status, promote, demote", cmd )); + ( *pargc )--; + ( *pargv )++; + + if( FD_UNLIKELY( args->failover.yes && args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_STATUS ) ) { + FD_LOG_ERR(( "--yes is only meaningful for `failover promote` and `failover demote`" )); + } + if( FD_UNLIKELY( args->failover.force && args->failover.cmd!=(int)FD_ADMINCTL_FAILOVER_CMD_PROMOTE ) ) { + FD_LOG_ERR(( "--force is only meaningful for `failover promote`" )); + } + if( FD_UNLIKELY( ( address || port ) && args->failover.cmd!=(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) ) { + FD_LOG_ERR(( "--address and --port are only meaningful for `failover promote` without --force" )); + } + if( FD_UNLIKELY( address && !fd_dns_resolve_address( address, &args->failover.addr ) ) ) { + FD_LOG_ERR(( "could not resolve --address `%s` to an IPv4 address", address )); + } + if( FD_UNLIKELY( address && !args->failover.addr ) ) { + FD_LOG_ERR(( "--address `%s` is 0.0.0.0, provide the active's own address", address )); + } + if( port ) { + uint value = 0U; + for( char const * c=port; *c; c++ ) { + uint digit = (uint)( *c-'0' ); + if( FD_UNLIKELY( digit>9U || value*10U+digit>65535U ) ) { + FD_LOG_ERR(( "--port `%s` must be a port number from 1 to 65535", port )); + } + value = value*10U+digit; + } + if( FD_UNLIKELY( !value ) ) { + FD_LOG_ERR(( "--port `%s` must be a port number from 1 to 65535", port )); + } + args->failover.port = (ushort)value; + } +} + +static char const * +role_name( uchar role ) { + if( FD_LIKELY( role==FD_FAILOVER_ROLE_STANDBY ) ) return "standby"; + if( FD_LIKELY( role==FD_FAILOVER_ROLE_ACTIVE ) ) return "active"; + return "unknown"; +} + +static char const * +action_name( uchar action ) { + return actionfailover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_PROMOTE ) { + FD_LOG_STDOUT(( "This takes the staked identity on your word. No other machine may hold it or be\n" + "in a promotion, including any validator running this identity without failover.\n" + "Run `failover status` on the other machine: its local role must be standby with\n" + "no promotion in progress, or that machine must be stopped or otherwise unable to\n" + "vote. Never promote both machines at once. Votes the other machine cast that\n" + "this machine is not aware of are not accounted for.\n" + "WARNING: --force skips every check on the other machine. Use it only when that\n" + "machine cannot sign, or both machines may vote with the staked identity.\n" + "It also permits incomplete or empty vote history, which may lose earlier lockouts.\n" + "Promotion tries eligible saved vote state first, then the vote account, and if the\n" + "vote account cannot be adopted it starts from an EMPTY vote history.\n" )); + } + if( FD_UNLIKELY( !args->failover.yes ) ) { + if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) { + FD_LOG_STDOUT(( "This asks the active to hand the staked identity to this standby. The active\n" + "keeps it if this standby is more than 64 slots behind its last vote. Otherwise\n" + "it gives up the identity first, so if replay here stalls before it catches up\n" + "the handoff fails and neither machine votes until `failover promote --force`.\n" + "Check that this machine is caught up. Type yes to continue: " )); + } else if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_DEMOTE ) { + FD_LOG_STDOUT(( "This gives up the staked identity and tells the other machine nothing. Nothing\n" + "votes until `failover promote --force` runs on one machine. To move the\n" + "identity to the other machine, run `failover promote` there instead. Type yes\n" + "to continue: " )); + } else { + FD_LOG_STDOUT(( "Type yes to continue: " )); + } + char line[ 16 ] = {0}; + if( FD_UNLIKELY( !fgets( line, sizeof(line), stdin ) ) ) FD_LOG_ERR(( "no confirmation given" )); + if( FD_UNLIKELY( strcmp( line, "yes\n" ) ) ) FD_LOG_ERR(( "not confirmed, nothing was done" )); + } +} + +/* What this machine knows of another holder of the staked identity, + from the checks that `failover promote --force` skips. It is not a go + ahead. */ +static char const * +other_holder( ulong result ) { + switch( result ) { + case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: return "n/a, this machine holds the identity"; + case FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE: return "the authenticated peer says it holds the identity"; + case FD_FAILOVER_CONTROL_RESULT_TAKEN: return "the peer took our last handoff and may be voting"; + case FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING: return "the peer has not answered our last handoff and may hold the identity"; + case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "gossip showed the staked identity at another host within 15 seconds, an active is probably voting, " + "for 15 seconds after a restart this can be an old entry"; + case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: return "unknown while a transition or key switch runs here"; + case FD_ADMINCTL_RESULT_SUCCESS: + case FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED: return "none seen by this machine, which does not prove the other machine is stopped, check it before " + "`failover promote --force`"; + default: return "unknown"; + } +} + +/* Prints failover status. */ +static void +failover_status_print( fd_adminctl_failover_status_resp_t const * resp ) { + if( FD_UNLIKELY( !resp->enabled ) ) { + FD_LOG_STDOUT(( "%-22s %s\n", "failover:", "disabled" )); + return; + } + + FD_LOG_STDOUT(( "%-22s %s\n", "failover:", "enabled" )); + FD_LOG_STDOUT(( "%-22s %s\n", "role:", role_name ( resp->role ) )); + if( FD_UNLIKELY( resp->request_paused ) ) { + FD_LOG_STDOUT(( "%-22s %s, paused at its deadline and not dialing, `failover promote` resumes it, with --yes on the active\n", "action:", action_name( resp->action ) )); + } else { + FD_LOG_STDOUT(( "%-22s %s\n", "action:", action_name( resp->action ) )); + } + FD_LOG_STDOUT(( "%-22s %s\n", "stuck:", !resp->stuck ? "no" : resp->request_paused ? "yes, the handoff request is paused, see the log" : "yes, see the log" )); + FD_LOG_STDOUT(( "%-22s %s\n", "link:", resp->link_statelink_state ] : "unknown" )); + FD_LOG_STDOUT(( "%-22s %s\n", "peer role:", resp->peer_role_valid ? role_name( resp->peer_role ) : "unknown, no authenticated failover session" )); + if( FD_UNLIKELY( !resp->peer_boot_id ) ) FD_LOG_STDOUT(( "%-22s none paired since boot\n", "peer boot id:" )); + else FD_LOG_STDOUT(( "%-22s %016lx\n", "peer boot id:", resp->peer_boot_id )); + if( FD_UNLIKELY( !resp->peer_addr && resp->role==FD_FAILOVER_ROLE_ACTIVE ) ) { + FD_LOG_STDOUT(( "%-22s n/a, this machine is the active\n", "peer address:" )); + } else if( FD_UNLIKELY( !resp->peer_addr ) ) { + FD_LOG_STDOUT(( "%-22s unknown, gossip has no contact info for the staked identity from another host\n", "peer address:" )); + } else { + FD_LOG_STDOUT(( "%-22s " FD_IP4_ADDR_FMT ":%u, from %s\n", "peer address:", FD_IP4_ADDR_FMT_ARGS( resp->peer_addr ), (uint)resp->peer_port, + resp->peer_addr_cmd ? "--address" : "gossip" )); + } + + if( FD_UNLIKELY( !resp->handoff_id ) ) { + FD_LOG_STDOUT(( "%-22s none since boot\n", "last handoff:" )); + } else { + FD_LOG_STDOUT(( "%-22s %lu, %s\n", "last handoff:", resp->handoff_id, + resp->handoff_resulthandoff_result ] : "unknown" )); + } + + FD_LOG_STDOUT(( "%-22s %s\n", "other holder:", other_holder( resp->promote_result ) )); + FD_LOG_STDOUT(( "%-22s %s\n", "takeover adopts:", + resp->promote_sourcepromote_source ] : "unknown" )); + if( FD_UNLIKELY( resp->promote_floor==FD_FAILOVER_SLOT_NULL ) ) FD_LOG_STDOUT(( "%-22s none known\n", "coverage floor:" )); + else FD_LOG_STDOUT(( "%-22s slot %lu\n", "coverage floor:", resp->promote_floor )); +} + +/* Signal handler that tells the operator a command is still waiting. */ +static void +wait_note( int sig ) { + (void)sig; + static char const note[] = "the validator has not answered for 3 seconds, it may be switching its identity, " + "see `failover: switching the identity` in its log, still waiting\n"; + ssize_t written = write( STDERR_FILENO, note, sizeof(note)-1UL ); + (void)written; +} + +/* Sends req to the failover tile and waits for the answer, which lands + in resp. Returns the adminctl result. */ +static ulong +failover_request( fd_adminctl_t * adminctl, + fd_adminctl_failover_req_t const * req, + void * resp, + ulong resp_max, + ulong * resp_sz ) { + void * payload = NULL; + ulong payload_max = 0UL; + ulong slot_idx = fd_adminctl_reserve( adminctl, &payload, &payload_max ); + if( FD_UNLIKELY( slot_idx==ULONG_MAX ) ) FD_LOG_ERR(( "all admin command slots are busy" )); + if( FD_UNLIKELY( sizeof(*req)>payload_max ) ) FD_LOG_ERR(( "adminctl failover payload too large" )); + fd_memcpy( payload, req, sizeof(*req) ); + fd_adminctl_publish( adminctl, slot_idx, FD_ADMINCTL_CMD_FAILOVER, sizeof(*req) ); + + struct sigaction sa = { .sa_handler = wait_note }; + if( FD_UNLIKELY( sigaction( SIGALRM, &sa, NULL ) ) ) FD_LOG_ERR(( "sigaction(SIGALRM) failed (%i-%s)", errno, fd_io_strerror( errno ) )); + alarm( 3U ); + ulong result = fd_adminctl_wait_response( adminctl, slot_idx, resp, resp_max, resp_sz ); + alarm( 0U ); + return result; +} + +static void __attribute__((noreturn)) +fail_disabled( int status ) { + if( status ) FD_LOG_ERR(( "this validator has no failover command bus, so it cannot report failover status" )); + FD_LOG_ERR(( "failover is not enabled on the running validator, set [failover.enabled] to true and restart it" )); +} + +static void __attribute__((noreturn)) +fail_incompatible( void ) { + FD_LOG_ERR(( "running validator returned an incompatible failover response, use the firedancer binary the validator runs" )); +} + +/* Before we prompt we check that failover is on and that this machine is + in the role the command needs, so nobody confirms a command that is + then refused. */ +static void +failover_precheck( fd_adminctl_t * adminctl, + args_t const * args ) { + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_STATUS }; + fd_adminctl_failover_status_resp_t status; + ulong status_sz = 0UL; + ulong result = failover_request( adminctl, &req, &status, sizeof(status), &status_sz ); + if( FD_UNLIKELY( result==FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ) ) fail_disabled( 0 ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS || status_sz!=sizeof(status) || + status.version!=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION ) ) return; + if( FD_UNLIKELY( status.enabled!=1 ) ) fail_disabled( 0 ); + int want_active = args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_DEMOTE; + if( FD_UNLIKELY( (status.role==FD_FAILOVER_ROLE_ACTIVE)!=want_active ) ) { + FD_LOG_ERR(( "`failover %s` refused, %s", fd_adminctl_failover_cmd_typed( (ulong)args->failover.cmd ), + control_result_name( FD_FAILOVER_CONTROL_RESULT_BAD_ROLE ) )); + } +} + +/* Send the command to the failover tile and print the result. */ +static void +failover_cmd_fn( args_t * args, + config_t * config ) { + fd_adminctl_t * adminctl = adminctl_client_attach( config, args->failover.name ); + int status = args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_STATUS; + if( !status && !args->failover.yes ) failover_precheck( adminctl, args ); + if( !status ) failover_confirm( args ); + + /* --yes only skips the prompt, the operator still has to give --force + explicitly. */ + fd_adminctl_failover_req_t req; + fd_memset( &req, 0, sizeof(req) ); + req.version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION; + req.cmd = (ulong)args->failover.cmd; + if( FD_UNLIKELY( args->failover.yes ) ) req.flags |= FD_ADMINCTL_FAILOVER_FLAG_YES; + if( FD_UNLIKELY( args->failover.force ) ) req.flags |= FD_ADMINCTL_FAILOVER_FLAG_FORCE; + req.addr = args->failover.addr; + req.port = args->failover.port; + + union { + fd_adminctl_failover_control_resp_t control; + fd_adminctl_failover_status_resp_t status; + } resp = { 0 }; + ulong resp_sz = 0UL; + ulong result = failover_request( adminctl, &req, &resp, sizeof(resp), &resp_sz ); + char const * cmd_name = fd_adminctl_failover_cmd_typed( (ulong)args->failover.cmd ); + char const * refusal = control_result_name( result ); + if( FD_UNLIKELY( refusal ) ) FD_LOG_ERR(( "`failover %s` refused, %s", cmd_name, refusal )); + switch( result ) { + case FD_ADMINCTL_RESULT_SUCCESS: + if( FD_UNLIKELY( resp_sz!=(status ? sizeof(resp.status) : sizeof(resp.control)) || + (status ? resp.status.version : resp.control.version)!=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION || + (status && resp.status.mode>=FD_FAILOVER_MODE_CNT) ) ) { + fail_incompatible(); + } + if( status ) { + failover_status_print( &resp.status ); + } else { + ulong id = resp.control.handoff_id; + FD_LOG_STDOUT(( "%-22s accepted\n", cmd_name )); + FD_LOG_STDOUT(( "%-22s %s\n", "role:", role_name ( resp.control.role ) )); + FD_LOG_STDOUT(( "%-22s %s\n", "action:", action_name( resp.control.action ) )); + if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) { + FD_LOG_STDOUT(( "%-22s %lu, done when `failover status` here shows role: active and last handoff: %lu, taken\n", "handoff:", id, id )); + } else if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_DEMOTE ) { + FD_LOG_STDOUT(( "%-22s %lu, done when `failover status` here shows role: standby and action: idle\n", "demotion:", id )); + } else { + FD_LOG_STDOUT(( "%-22s done when `failover status` here shows role: active and action: idle\n", "promotion:" )); + } + } + break; + case FD_FAILOVER_CONTROL_RESULT_BUSY: + FD_LOG_ERR(( "another failover command is in flight, try again" )); + case FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE: + if( status ) FD_LOG_ERR(( "the validator's failover tile did not answer in time" )); + FD_LOG_ERR(( "the validator's failover tile did not answer in time, the outcome is indeterminate, run `failover status`" )); + case FD_ADMINCTL_RESULT_UNKNOWN_COMMAND: + if( status ) { + FD_LOG_STDOUT(( "%-22s %s\n", "failover:", "disabled" )); + break; + } + fail_disabled( 0 ); + case FD_ADMINCTL_RESULT_UNSUPPORTED: + fail_disabled( status ); + case FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH: + case FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH: + FD_LOG_ERR(( "failover is incompatible with the running validator, use the firedancer binary the validator runs" )); + default: + FD_LOG_ERR(( "unexpected failover result %lu", result )); + } +} + +static void +failover_args_help( fd_action_help_t * help ) { + fd_action_help_arg( help, "--name", "", "Name of the validator instance to attach to, if more than one is\n" + "running on this host" ); + fd_action_help_arg( help, "--yes", NULL, "Skip the confirmation prompt. It does not authorize peer or\n" + "vote-history overrides" ); + fd_action_help_arg( help, "--force", NULL, "Only with `promote`. Take the identity without the active, skip\n" + "every check on the other machine and stop waiting for its answer\n" + "to our handoff. Use it only when the other machine cannot sign.\n" + "Also accept incomplete or empty vote history, earlier lockouts may\n" + "be lost" ); + fd_action_help_arg( help, "--address", "
", "Only with `promote` without --force. The active to ask, an IPv4\n" + "address or host name, instead of the address gossip shows for it,\n" + "for example when the active listens on a private network" ); + fd_action_help_arg( help, "--port", "", "Only with `promote` without --force. The active's failover port.\n" + "Without it we dial our own [failover.listen_port], so both machines\n" + "normally use the same port" ); +} + +action_t fd_action_failover = { + .name = "failover", + .args = failover_cmd_args, + .fn = failover_cmd_fn, + .require_config = 0, + .perm = NULL, + .description = "Inspect and drive failover of the running validator", + .detail = "Hands off the staked identity between two validators. A validator with\n" + "[failover] enabled boots as a standby under its junk identity and does\n" + "not vote until promoted.\n" + "\n" + "`status` prints this machine's failover state. `promote` on the standby\n" + "asks the active to hand over its identity. `promote --force` takes the\n" + "identity without the active, for when it cannot sign. Each asks for\n" + "confirmation unless --yes is given.\n" + "\n" + "This command does not start a validator; it attaches to one that is already\n" + "running. With no arguments it discovers the running validator automatically.\n" + "If multiple validators are running, pass --name to select one.\n", + .usage = "failover status|promote|demote [--name ] [--yes] [--force] [--address
] [--port ]", + .args_help = failover_args_help, +}; diff --git a/src/app/firedancer/main.c b/src/app/firedancer/main.c index adae39a2259..194cc09a21b 100644 --- a/src/app/firedancer/main.c +++ b/src/app/firedancer/main.c @@ -190,6 +190,7 @@ extern action_t fd_action_add_authorized_voter; extern action_t fd_action_remove_all_authorized_voters; extern action_t fd_action_set_identity; extern action_t fd_action_get_identity; +extern action_t fd_action_failover; extern action_t fd_action_ps; extern action_t fd_action_monitor_gossip; extern action_t fd_action_snapshot_create; @@ -214,6 +215,7 @@ action_t * ACTIONS[] = { &fd_action_monitor_gossip, &fd_action_set_identity, &fd_action_get_identity, + &fd_action_failover, &fd_action_ps, &fd_action_snapshot_create, &fd_action_wait, diff --git a/src/app/shared/fd_action.h b/src/app/shared/fd_action.h index 6ece0a79db4..4e042c5bdea 100644 --- a/src/app/shared/fd_action.h +++ b/src/app/shared/fd_action.h @@ -61,6 +61,15 @@ union fdctl_args { char name[ 64UL ]; } get_identity; + struct { + char name[ 64UL ]; + int cmd; + int yes; + int force; + uint addr; + ushort port; + } failover; + struct { int clean; } ps; From 563396182fe96f795354dc3c2735d8f6992c6ed9 Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Thu, 8 Oct 2026 23:02:33 +0000 Subject: [PATCH 11/23] failover: document the controller, channel and protocol states --- src/discof/admin/fd_adminctl.h | 8 +-- src/discof/failover/fd_failover_channel.c | 42 ++++++++---- src/discof/failover/fd_failover_proto.c | 14 +++- src/discof/failover/fd_failover_proto.h | 28 ++++---- src/discof/failover/fd_failover_tile.c | 80 +++++++++++++++++++++-- src/discof/failover/fd_failover_tls.c | 10 ++- 6 files changed, 142 insertions(+), 40 deletions(-) diff --git a/src/discof/admin/fd_adminctl.h b/src/discof/admin/fd_adminctl.h index 4f82f3555ab..cf6f0db53ae 100644 --- a/src/discof/admin/fd_adminctl.h +++ b/src/discof/admin/fd_adminctl.h @@ -173,7 +173,7 @@ FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_req_t )==32UL, failover_re FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)==24UL, failover_control_resp_v1_layout ); /* What the failover controller is doing right now. */ -#define FD_FAILOVER_ACTION_IDLE (0UL) +#define FD_FAILOVER_ACTION_IDLE (0UL) /* no local transition is running */ #define FD_FAILOVER_ACTION_DEMOTE_SWITCH (1UL) /* waiting for the junk key to be installed */ #define FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK (2UL) /* DEMOTED sent, waiting for the peer */ #define FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY (3UL) /* waiting for replay to reach the tower tip */ @@ -190,8 +190,8 @@ FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)==24UL, failover_co #define FD_FAILOVER_SOURCE_CNT (2UL) /* The outcome of the last handoff this machine requested or gave. */ -#define FD_FAILOVER_HANDOFF_NONE (0UL) -#define FD_FAILOVER_HANDOFF_PENDING (1UL) +#define FD_FAILOVER_HANDOFF_NONE (0UL) /* nothing requested or sent */ +#define FD_FAILOVER_HANDOFF_PENDING (1UL) /* requested or sent, no answer yet */ #define FD_FAILOVER_HANDOFF_TAKEN (2UL) /* the standby took the identity */ #define FD_FAILOVER_HANDOFF_DECLINED (3UL) /* the standby did not take the identity, or the old active did not record that it did */ #define FD_FAILOVER_HANDOFF_RESTARTED (4UL) /* the peer came back with a new boot_id */ @@ -202,7 +202,7 @@ FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)==24UL, failover_co /* `failover status`, provided by the failover tile. */ struct fd_adminctl_failover_status_resp_v1 { ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ - uchar enabled; /* 0 when failover is off, the other fields keep their init values then */ + uchar enabled; /* 1 from the failover tile */ uchar role; /* FD_FAILOVER_ROLE_* */ uchar action; /* FD_FAILOVER_ACTION_* */ uchar stuck; /* a transition failed or is overdue */ diff --git a/src/discof/failover/fd_failover_channel.c b/src/discof/failover/fd_failover_channel.c index 547333bbe79..3abe87bf90c 100644 --- a/src/discof/failover/fd_failover_channel.c +++ b/src/discof/failover/fd_failover_channel.c @@ -12,16 +12,16 @@ #include #define SOURCE_CNT (256UL) -#define PHASE_CONNECT (0) -#define PHASE_TLS (1) -#define PHASE_HELLO (2) -#define PHASE_READY (3) +#define PHASE_CONNECT (0) /* Wait for outbound TCP to connect. */ +#define PHASE_TLS (1) /* Complete the TLS handshake. */ +#define PHASE_HELLO (2) /* Read and validate the peer HELLO. */ +#define PHASE_READY (3) /* Drain our HELLO before pairing. */ /* A HELLO whose junk key does not match the key TLS authenticated */ #define HELLO_ERR_TLS_KEY (-1) struct candidate { - int fd; + int fd; /* -1 marks an unused candidate. */ int phase; int dialed; uint address; @@ -85,6 +85,7 @@ void * fd_failover_channel_new( void * shmem ) { if( !shmem || !fd_ulong_is_aligned( (ulong)shmem, fd_failover_channel_align() ) ) return NULL; fd_failover_channel_t * ch = shmem; + /* Initialize the listening state with no sockets open. */ fd_memset( ch, 0, sizeof(*ch) ); ch->listen_fd = -1; ch->paired_idx = -1; @@ -111,6 +112,7 @@ static void close_candidate( struct candidate * c ) { fd_failover_tls_fini( &c->tls ); if( c->fd!=-1 ) close( c->fd ); + /* Closing a candidate frees its slot. */ c->fd = -1; c->dialed = 0; c->rx_used = c->tx_used = c->tx_sent = 0UL; @@ -120,6 +122,7 @@ close_candidate( struct candidate * c ) { static void reset( fd_failover_channel_t * ch ) { for( ulong i=0; icandidates[i] ); + /* Discard the paired session. */ ch->paired_idx = -1; ch->last_rx = 0L; fd_memset( &ch->peer_hello, 0, sizeof(ch->peer_hello) ); @@ -135,9 +138,10 @@ want_dial( fd_failover_channel_t const * ch ) { BACKOFF with an immediate retry with one. */ static void rest( fd_failover_channel_t * ch ) { - if( ch->paired_idx>=0 ) return; + if( ch->paired_idx>=0 ) return; /* Keep a paired session through address changes. */ for( ulong i=0; icandidates[i].dialed ) close_candidate( &ch->candidates[i] ); ch->dial_peer = want_dial( ch ); + /* An unpaired channel listens or becomes ready to dial. */ ch->state = fd_failover_session_init( ch->dial_peer ); ch->retry_at = 0L; } @@ -182,6 +186,7 @@ fd_failover_channel_set_identity( fd_failover_channel_t * ch, if( !fd_memeq( junk_pubkey, hello->junk_pubkey, 32UL ) || fd_memeq( hello->junk_pubkey, hello->staked_pubkey, 32UL ) ) return -1; reset( ch ); + /* An identity change requires a fresh session. */ ch->state = fd_failover_session_init( ch->dial_peer ); ch->self_hello = *hello; ch->member_cert_set = 0; @@ -280,7 +285,9 @@ drop( fd_failover_channel_t * ch, /* A candidate from the listener does not move the session until it pairs. */ if( !paired && accepted ) return; + /* Loss returns the session to listening or backoff. */ ch->state = fd_failover_session_step( ch->state, ch->dial_peer, event ); + /* After loss, use the current handoff address to choose listener or dialer. */ if( paired && want_dial( ch )!=ch->dial_peer ) rest( ch ); if( ch->dial_peer ) { /* A session counts as established once it outlives the handshake @@ -435,6 +442,7 @@ start( fd_failover_channel_t * ch, long now, int phase ) { struct candidate * c = &ch->candidates[idx]; + /* Outbound sockets start at CONNECT, accepted sockets start at TLS. */ c->fd = fd; c->address = address; c->phase = phase; c->dialed = phase==PHASE_CONNECT; c->deadline = fd_long_sat_add( now, ch->hello_timeout ); fd_failover_wire_session_init( &c->wire ); @@ -464,6 +472,7 @@ accept_candidates( fd_failover_channel_t * ch, close( fd ); continue; } + /* An accepted candidate leaves the session unpaired until authentication. */ if( !start( ch, idx, fd, addr.sin_addr.s_addr, now, PHASE_TLS ) ) ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_PEER_CONNECTED ); } @@ -578,15 +587,17 @@ connect_candidate( fd_failover_channel_t * ch, struct candidate * c = &ch->candidates[idx]; struct sockaddr_in addr = { .sin_family=AF_INET, .sin_port=fd_ushort_bswap( ch->peer_port ), .sin_addr.s_addr=ch->peer_addr }; if( connect( c->fd, fd_type_pun( &addr ), sizeof(addr) ) && errno!=EISCONN ) { - if( errno==EINPROGRESS || errno==EALREADY || errno==EINTR ) return 0; + if( errno==EINPROGRESS || errno==EALREADY || errno==EINTR ) return 0; /* TCP is still connecting. */ if( now>=ch->dial_log_at[0] ) { ch->dial_log_at[0] = fd_long_sat_add( now, 60000000000L ); FD_LOG_WARNING(( "could not connect to the failover peer at `" FD_IP4_ADDR_FMT ":%hu` (%i-%s), check that it runs and that the machines can reach each other on [failover.listen_port]", FD_IP4_ADDR_FMT_ARGS( ch->peer_addr ), ch->peer_port, errno, fd_io_strerror( errno ) )); } + /* A failed connect returns the dialer to backoff. */ drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); return 0; } + /* TCP connected, begin TLS and enter the session HELLO state. */ c->phase = PHASE_TLS; ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_CONNECTED ); return 1; @@ -607,11 +618,13 @@ handshake_candidate( fd_failover_channel_t * ch, FD_LOG_WARNING(( "TLS handshake with `" FD_IP4_ADDR_FMT "` failed (%s), not necessarily the other failover machine, check peer reachability and junk keys on both machines", FD_IP4_ADDR_FMT_ARGS( c->address ), reason ? fd_tls_reason_cstr( reason ) : "connection closed or socket error" )); } + /* TLS failure closes the candidate and returns an outbound dial to backoff. */ drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); return 0; } - if( !rc ) return 0; + if( !rc ) return 0; /* Keep waiting for TLS. */ *busy = 1; + /* TLS authenticated a key, now check the peer HELLO. */ c->phase = PHASE_HELLO; /* The dialer sends first. On the listener we respond only to a HELLO that passed the checks below. */ @@ -629,12 +642,13 @@ read_candidate_hello( fd_failover_channel_t * ch, struct candidate * c = &ch->candidates[idx]; ushort type; ulong sz; - if( read_frame( ch, idx, now, busy, &type, payload, &sz )<=0 ) return 0; + if( read_frame( ch, idx, now, busy, &type, payload, &sz )<=0 ) return 0; /* Wait for HELLO, or stop after a read failure. */ if( type!=FD_FAILOVER_MSG_HELLO || sz!=sizeof(c->hello) ) { if( now>=ch->hello_log_at[0] ) { ch->hello_log_at[0] = fd_long_sat_add( now, 60000000000L ); FD_LOG_WARNING(( "rejected the failover HELLO from `" FD_IP4_ADDR_FMT "`, its first frame is not a HELLO, check who can reach the failover port", FD_IP4_ADDR_FMT_ARGS( c->address ) )); } + /* A malformed HELLO closes this candidate without pairing. */ drop( ch, idx, now, FD_FAILOVER_EV_HELLO_FATAL ); return 0; } @@ -649,14 +663,16 @@ read_candidate_hello( fd_failover_channel_t * ch, ch->hello_log_at[bucket] = fd_long_sat_add( now, 60000000000L ); FD_LOG_WARNING(( "rejected the failover HELLO from `" FD_IP4_ADDR_FMT "`, %s", FD_IP4_ADDR_FMT_ARGS( c->address ), hello_err_name( err ) )); } + /* A refused HELLO closes this candidate without pairing. */ drop( ch, idx, now, FD_FAILOVER_EV_HELLO_FATAL ); return 0; } + /* The peer is authenticated, finish any HELLO output before pairing. */ c->phase = PHASE_READY; if( !c->dialed ) { c->tx_used = fd_failover_wire_encode( &c->wire, c->tx, FD_FAILOVER_MSG_HELLO, (uchar const *)&ch->self_hello, sizeof(ch->self_hello) ); - if( flush( ch, idx, now, busy )!=0 ) return 0; + if( flush( ch, idx, now, busy )!=0 ) return 0; /* Remain READY until output drains, unless the write fails. */ } return 1; } @@ -671,12 +687,14 @@ pair_candidate( fd_failover_channel_t * ch, listener while we were dialing pairs as the listener. */ for( ulong i=0; icandidates[i] ); if( !c->dialed && ch->dial_peer ) { + /* An inbound candidate won, pair as the listener. */ ch->dial_peer = 0; ch->state = fd_failover_session_init( ch->dial_peer ); } ch->paired_idx = (int)idx; c->tls.paired = 1; ch->peer_hello = c->hello; + /* HELLO is complete, this candidate becomes the paired session. */ ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_HELLO_OK ); ch->generation++; ch->last_rx = now; @@ -707,7 +725,7 @@ service_candidate( fd_failover_channel_t * ch, if( c->phase==PHASE_TLS ) { if( !handshake_candidate( ch, idx, now, busy ) ) return; } - if( flush( ch, idx, now, busy )!=0 ) return; + if( flush( ch, idx, now, busy )!=0 ) return; /* Wait for output, or stop after a write failure. */ if( c->phase==PHASE_HELLO ) { if( !read_candidate_hello( ch, idx, now, busy, payload ) ) return; } @@ -751,6 +769,7 @@ fd_failover_channel_poll( fd_failover_channel_t * ch, FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` did not finish the handshake in time, check that it runs and that the machines can reach each other on [failover.listen_port]", FD_IP4_ADDR_FMT_ARGS( c->address ) )); } + /* An expired candidate closes, an outbound dial returns to backoff. */ drop( ch, i, now, FD_FAILOVER_EV_TIMEOUT ); *busy = 1; } } @@ -765,6 +784,7 @@ fd_failover_channel_poll( fd_failover_channel_t * ch, int fd = socket( AF_INET, SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC, 0 ); *busy = 1; if( fd==-1 ) { drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); return delivered; } + /* Backoff expired and an address is available, start dialing. */ ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_RETRY ); start( ch, idx, fd, ch->peer_addr, now, PHASE_CONNECT ); } diff --git a/src/discof/failover/fd_failover_proto.c b/src/discof/failover/fd_failover_proto.c index 836752325c4..0a1f90ec375 100644 --- a/src/discof/failover/fd_failover_proto.c +++ b/src/discof/failover/fd_failover_proto.c @@ -46,6 +46,7 @@ fd_failover_member_cert_check( fd_failover_hello_t const * hello, ulong fd_failover_session_init( int dial_peer ) { + /* Start in backoff for a dialer, listening otherwise. */ return dial_peer ? FD_FAILOVER_SESSION_BACKOFF : FD_FAILOVER_SESSION_LISTENING; } @@ -53,6 +54,7 @@ ulong fd_failover_session_step( ulong state, int dial_peer, int event ) { + /* Invalid inputs leave the state unchanged. */ if( FD_UNLIKELY( state>=FD_FAILOVER_SESSION_CNT ) ) return state; if( FD_UNLIKELY( dial_peer<0 || dial_peer>1 ) ) return state; if( FD_UNLIKELY( event<0 || event>=FD_FAILOVER_EV_CNT ) ) return state; @@ -63,32 +65,40 @@ fd_failover_session_step( ulong state, /* Candidates come and go without changing the listener's state. */ switch( state ) { case FD_FAILOVER_SESSION_LISTENING: + /* A valid HELLO pairs the listener. */ if( event==FD_FAILOVER_EV_HELLO_OK ) return FD_FAILOVER_SESSION_PAIRED; break; case FD_FAILOVER_SESSION_PAIRED: + /* A lost session returns to listening. */ if( lost ) return FD_FAILOVER_SESSION_LISTENING; break; } - return state; + return state; /* Other events leave the listener unchanged. */ } switch( state ) { case FD_FAILOVER_SESSION_BACKOFF: + /* Backoff expired, start another dial. */ if( event==FD_FAILOVER_EV_RETRY ) return FD_FAILOVER_SESSION_DIALING; break; case FD_FAILOVER_SESSION_DIALING: + /* TCP connected, start authentication. */ if( event==FD_FAILOVER_EV_CONNECTED ) return FD_FAILOVER_SESSION_HELLO; + /* A failed or lost connection waits for a retry. */ if( lost ) return FD_FAILOVER_SESSION_BACKOFF; break; case FD_FAILOVER_SESSION_HELLO: + /* A valid HELLO pairs the dialer. */ if( event==FD_FAILOVER_EV_HELLO_OK ) return FD_FAILOVER_SESSION_PAIRED; + /* A failed or lost connection waits for a retry. */ if( lost ) return FD_FAILOVER_SESSION_BACKOFF; break; case FD_FAILOVER_SESSION_PAIRED: + /* A failed or lost connection waits for a retry. */ if( lost ) return FD_FAILOVER_SESSION_BACKOFF; break; } - return state; + return state; /* Other events leave the dialer unchanged. */ } int diff --git a/src/discof/failover/fd_failover_proto.h b/src/discof/failover/fd_failover_proto.h index 5a606778dfb..0fcb7dfda60 100644 --- a/src/discof/failover/fd_failover_proto.h +++ b/src/discof/failover/fd_failover_proto.h @@ -26,28 +26,28 @@ #define FD_FAILOVER_MODE_CNT (1U) /* Roles for each endpoint */ -#define FD_FAILOVER_ROLE_STANDBY (0UL) -#define FD_FAILOVER_ROLE_ACTIVE (1UL) +#define FD_FAILOVER_ROLE_STANDBY (0UL) /* Run under the junk identity. */ +#define FD_FAILOVER_ROLE_ACTIVE (1UL) /* Run under the staked identity. */ /* Session states for one authenticated TCP session per pair. The listener only ever shows LISTENING or PAIRED, its candidates are tracked per socket. The dialer walks BACKOFF, DIALING, HELLO and PAIRED. */ -#define FD_FAILOVER_SESSION_LISTENING (0UL) -#define FD_FAILOVER_SESSION_DIALING (1UL) -#define FD_FAILOVER_SESSION_HELLO (2UL) -#define FD_FAILOVER_SESSION_PAIRED (3UL) -#define FD_FAILOVER_SESSION_BACKOFF (4UL) +#define FD_FAILOVER_SESSION_LISTENING (0UL) /* Accept candidates until one authenticates. */ +#define FD_FAILOVER_SESSION_DIALING (1UL) /* Wait for outbound TCP to connect. */ +#define FD_FAILOVER_SESSION_HELLO (2UL) /* Complete TLS and exchange HELLO. */ +#define FD_FAILOVER_SESSION_PAIRED (3UL) /* Exchange messages with the authenticated peer. */ +#define FD_FAILOVER_SESSION_BACKOFF (4UL) /* Wait before the next dial attempt. */ #define FD_FAILOVER_SESSION_CNT (5UL) /* Session events, inputs to fd_failover_session_step */ -#define FD_FAILOVER_EV_PEER_CONNECTED (0) -#define FD_FAILOVER_EV_CONNECTED (1) -#define FD_FAILOVER_EV_HELLO_OK (2) -#define FD_FAILOVER_EV_HELLO_FATAL (3) -#define FD_FAILOVER_EV_TIMEOUT (4) -#define FD_FAILOVER_EV_LINK_LOST (5) -#define FD_FAILOVER_EV_RETRY (6) +#define FD_FAILOVER_EV_PEER_CONNECTED (0) /* An inbound TCP candidate was accepted. */ +#define FD_FAILOVER_EV_CONNECTED (1) /* Outbound TCP connected. */ +#define FD_FAILOVER_EV_HELLO_OK (2) /* The peer passed authentication and HELLO checks. */ +#define FD_FAILOVER_EV_HELLO_FATAL (3) /* The peer failed HELLO checks. */ +#define FD_FAILOVER_EV_TIMEOUT (4) /* The handshake or session deadline expired. */ +#define FD_FAILOVER_EV_LINK_LOST (5) /* The connection closed or failed. */ +#define FD_FAILOVER_EV_RETRY (6) /* The next dial attempt may start. */ #define FD_FAILOVER_EV_CNT (7) /* HELLO pairing outcomes */ diff --git a/src/discof/failover/fd_failover_tile.c b/src/discof/failover/fd_failover_tile.c index 3d271628453..b1063f07936 100644 --- a/src/discof/failover/fd_failover_tile.c +++ b/src/discof/failover/fd_failover_tile.c @@ -49,9 +49,9 @@ in fd_adminctl.h, `failover status` reports them. */ /* Which key a switch installs, CNT while none is in flight */ -#define FD_FAILOVER_SWITCH_KEY_JUNK (0UL) -#define FD_FAILOVER_SWITCH_KEY_STAKED (1UL) -#define FD_FAILOVER_SWITCH_KEY_CNT (2UL) +#define FD_FAILOVER_SWITCH_KEY_JUNK (0UL) /* The junk identity. */ +#define FD_FAILOVER_SWITCH_KEY_STAKED (1UL) /* The staked identity. */ +#define FD_FAILOVER_SWITCH_KEY_CNT (2UL) /* No identity switch is pending. */ /* Our DEMOTED to the handoff target */ #define FD_FAILOVER_DELIVERY_NONE (0UL) /* Nothing to send. */ @@ -326,7 +326,9 @@ privileged_init( fd_topo_t const * topo, FD_SCRATCH_ALLOC_INIT( l, scratch ); fd_failover_tile_ctx_t * ctx = FD_SCRATCH_ALLOC_APPEND( l, alignof(fd_failover_tile_ctx_t), sizeof(fd_failover_tile_ctx_t) ); void * channel_mem = FD_SCRATCH_ALLOC_APPEND( l, fd_failover_channel_align(), fd_failover_channel_footprint() ); + /* Start idle with no handoff or promotion in progress. */ fd_memset( ctx, 0, sizeof(fd_failover_tile_ctx_t) ); + /* No admin switch is pending at boot. */ ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT; ctx->deadline_slot = FD_FAILOVER_SLOT_NULL; ctx->peer_floor = FD_FAILOVER_SLOT_NULL; @@ -542,6 +544,7 @@ dial_port( fd_failover_tile_ctx_t const * ctx ) { static void set_role( fd_failover_tile_ctx_t * ctx, ulong role ) { + /* Publish the role reached by the completed identity switch. */ ctx->role = role; ctx->hello.role = (uchar)role; fd_failover_channel_set_role( ctx->channel, role ); @@ -880,6 +883,7 @@ request_switch( fd_failover_tile_ctx_t * ctx, ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_SWITCH_REQ, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub ); ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark ); + /* Keep the switch pending until admin responds to this request ID. */ ctx->id_switch.pending_key = key; ctx->id_switch.fresh = 0; return ctx->id_switch.request_id; @@ -896,6 +900,7 @@ switch_answer( fd_failover_tile_ctx_t * ctx, FD_LOG_WARNING(( "dropping a stale identity switch answer" )); return 0; } + /* A matching response ends the pending switch. */ ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT; ctx->id_switch.fresh = 1; return 1; @@ -906,6 +911,7 @@ static void handoff_resolved( fd_failover_tile_ctx_t * ctx, ulong result ) { if( FD_UNLIKELY( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ) ) ctx->tx.valid = 0; + /* The handoff outcome ends the ACK wait, this machine stays standby. */ if( FD_LIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) ctx->action = FD_FAILOVER_ACTION_IDLE; ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE; ctx->handoff.result = result; @@ -919,6 +925,7 @@ demotion_abort( fd_failover_tile_ctx_t * ctx ) { if( FD_UNLIKELY( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ) ) { handoff_result_to( ctx, &ctx->handoff.target, ctx->handoff.id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); } + /* End the failed demotion without changing the installed identity. */ ctx->action = FD_FAILOVER_ACTION_IDLE; ctx->stuck = 1; ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE; @@ -950,6 +957,7 @@ start_demotion( fd_failover_tile_ctx_t * ctx, ctx->stuck = 0; if( FD_UNLIKELY( !send ) ) FD_LOG_NOTICE(( "demotion %lu started without a handoff, giving up the staked identity, no DEMOTED will be sent", ctx->handoff.id )); demotion_deadline_start( ctx ); + /* Start the junk key switch before sending any final state. */ ctx->action = FD_FAILOVER_ACTION_DEMOTE_SWITCH; } @@ -967,6 +975,7 @@ final_tower_ok( fd_failover_tile_ctx_t const * ctx ) { demote requires shared-source recovery. */ static void demotion_switched( fd_failover_tile_ctx_t * ctx ) { + /* The junk key is installed and the Tower has drained. */ set_role( ctx, FD_FAILOVER_ROLE_STANDBY ); FD_LOG_NOTICE(( "%s %lu: the tower drained, we are a standby now", demotion_kind( ctx ), ctx->handoff.id )); @@ -979,10 +988,12 @@ demotion_switched( fd_failover_tile_ctx_t * ctx ) { if( FD_UNLIKELY( !final_ok ) ) { /* Without final state, recovery needs the vote account. */ FD_LOG_WARNING(( "demotion %lu has no final tower for its last vote %lu, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->last_vote_slot )); + /* End demotion without sending final state. */ demotion_abort( ctx ); return; } FD_LOG_NOTICE(( "demotion %lu is done, neither machine holds the staked identity now, nothing votes until `failover promote --force` runs on one of them", ctx->handoff.id )); + /* A standalone demotion finishes here. */ ctx->action = FD_FAILOVER_ACTION_IDLE; return; } @@ -996,6 +1007,7 @@ demotion_switched( fd_failover_tile_ctx_t * ctx ) { tower of our last vote. The peer gets no final state and cannot promote. */ FD_LOG_WARNING(( "handoff %lu has no final tower for its last vote %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->last_vote_slot )); + /* End demotion without sending final state. */ demotion_abort( ctx ); return; } @@ -1004,6 +1016,7 @@ demotion_switched( fd_failover_tile_ctx_t * ctx ) { /* The control slot may still be holding an unsent response, so if this fails we retry from the wait. */ (void)queue_demoted( ctx ); + /* Final state is ready, wait for the bound peer to accept or refuse it. */ ctx->action = FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK; demotion_deadline_start( ctx ); } @@ -1111,6 +1124,7 @@ reject_promotion( fd_failover_tile_ctx_t * ctx, int bad_tower ) { if( FD_UNLIKELY( bad_tower && ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ) ctx->peer_tower.valid = 0; ctx->stuck = 1; + /* Failed local promotion returns to idle on the standby. */ ctx->action = FD_FAILOVER_ACTION_IDLE; if( FD_UNLIKELY( !ctx->promotion.from_peer ) ) { FD_LOG_WARNING(( "%s: promotion failed (%s), we stay a standby, check that one machine votes and see `failover status`, " @@ -1119,6 +1133,7 @@ reject_promotion( fd_failover_tile_ctx_t * ctx, } ctx->peer_tower.valid = 0; ctx->promotion.from_peer = 0; + /* A failed handoff waits for the old active to confirm our refusal. */ request_wait_result( ctx ); if( FD_UNLIKELY( reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), we stay a standby, another machine may hold the staked identity, " @@ -1173,6 +1188,7 @@ start_promotion( fd_failover_tile_ctx_t * ctx, ctx->last_vote_slot = ctx->promotion.adopt.tip; ctx->stuck = 0; deadline_start( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + /* Replay must reach the selected history before adoption. */ ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY; if( FD_UNLIKELY( from_peer && ctx->promotion.adopt.tip!=FD_FAILOVER_SLOT_NULL && ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || ctx->replay_slotpromotion.adopt.tip ) ) ) { @@ -1192,9 +1208,11 @@ promote_fallback( fd_failover_tile_ctx_t * ctx, FD_LOG_NOTICE(( "%s: %s is not eligible (%s), trying the vote account", ctx->promotion.label, source_name( ctx->promotion.source ), reason )); if( ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ctx->peer_tower.valid = 0; + /* Ineligible saved state falls back to the vote account. */ ctx->promotion.source = FD_FAILOVER_SOURCE_VOTE_ACCOUNT; } else { FD_LOG_WARNING(( "%s: the vote account cannot be adopted (%s), --force proceeds with an empty tower", ctx->promotion.label, reason )); + /* Only a forced operator promotion may fall back to empty history. */ ctx->promotion.empty = 1; } ctx->promotion.adopt.valid = 0; @@ -1204,6 +1222,7 @@ promote_fallback( fd_failover_tile_ctx_t * ctx, ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; ctx->promotion.retry = 0; ctx->adopt_result_fresh = 0; + /* Retry adoption from the fallback source after the prior attempt has ended. */ ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY; return 1; } @@ -1263,6 +1282,7 @@ step_demote_switch( fd_failover_tile_ctx_t * ctx, /* The switch request did not go out and we still have the identity, so we send no final state. */ FD_LOG_WARNING(( "an identity switch could not be requested, remaining the active" )); + /* End demotion without sending final state. */ demotion_abort( ctx ); return; } @@ -1281,6 +1301,7 @@ step_demote_switch( fd_failover_tile_ctx_t * ctx, ctx->id_switch.overdue = 0; FD_LOG_WARNING(( "the identity switch for %s %lu failed (%s), we keep the staked identity and stay the active", demotion_kind( ctx ), ctx->handoff.id, ctx->id_switch.result.result==FD_FAILOVER_SWITCH_ERR_DISABLED ? "failover disabled" : "unknown" )); + /* End demotion without sending final state. */ demotion_abort( ctx ); return; } @@ -1304,6 +1325,7 @@ step_demote_wait_ack( fd_failover_tile_ctx_t * ctx ) { FD_LOG_WARNING(( "the peer restarted before it answered handoff %lu and may have voted before it restarted, neither machine holds the staked identity now, " "nothing votes until `failover promote --force` runs on one of them", ctx->handoff.id )); ctx->peer_tower.valid = 0; + /* A new peer boot ends this ACK wait without proving whether it voted. */ handoff_resolved( ctx, FD_FAILOVER_HANDOFF_RESTARTED ); return; } @@ -1327,6 +1349,7 @@ step_promote_wait_replay( fd_failover_tile_ctx_t * ctx, long now = fd_failover_clock(); promote_deadline_arm( ctx, now ); if( FD_UNLIKELY( promote_expired( ctx, now ) ) ) { + /* Replay timed out, fail the promotion while remaining standby. */ reject_promotion( ctx, FD_FAILOVER_REJECT_REPLAY_BEHIND, 0 ); return; } @@ -1339,6 +1362,7 @@ step_promote_wait_replay( fd_failover_tile_ctx_t * ctx, } ulong id = publish_adopt_state( ctx, stem ); if( FD_UNLIKELY( id==ULONG_MAX ) ) { + /* End the failed adoption attempt while remaining standby. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } @@ -1346,6 +1370,7 @@ step_promote_wait_replay( fd_failover_tile_ctx_t * ctx, ctx->promotion.empty ? "an empty tower authorized by --force" : source_name( ctx->promotion.source ) )); ctx->promotion.adopt_id = id; ctx->promotion.retry = 0; + /* The adoption request is published, wait for its response. */ ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT; return; } @@ -1356,6 +1381,7 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, long now = fd_failover_clock(); promote_deadline_arm( ctx, now ); if( FD_UNLIKELY( promote_expired( ctx, now ) ) ) { + /* Adoption timed out, fail promotion while remaining standby. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } @@ -1365,19 +1391,23 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, if( FD_LIKELY( ctx->replay_slot<=ctx->promotion.retry_slot ) ) return; ulong id = publish_adopt_state( ctx, stem ); if( FD_UNLIKELY( id==ULONG_MAX ) ) { + /* End the failed adoption attempt while remaining standby. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } + /* Replay advanced, retry adoption without leaving the adoption wait. */ ctx->promotion.adopt_id = id; ctx->promotion.retry = 0; return; } - if( FD_LIKELY( !ctx->adopt_result_fresh ) ) return; + if( FD_LIKELY( !ctx->adopt_result_fresh ) ) return; /* Wait for an adoption response. */ ctx->adopt_result_fresh = 0; - if( FD_UNLIKELY( ctx->adopt_result_id!=ctx->promotion.adopt_id ) ) return; + if( FD_UNLIKELY( ctx->adopt_result_id!=ctx->promotion.adopt_id ) ) return; /* Ignore responses to earlier attempts. */ if( FD_UNLIKELY( ctx->adopt_result.result==FD_TOWER_ADOPT_ERR_UNREPLAYED ) ) { + /* Operator promotion tries a fallback before waiting for more replay. */ if( promote_fallback( ctx, "its blocks have not been replayed" ) ) return; if( FD_UNLIKELY( !ctx->promotion.from_peer ) ) { + /* End the failed adoption attempt while remaining standby. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } @@ -1385,14 +1415,17 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, FD_LOG_NOTICE(( "%s: the tower tile has not replayed every block the tower votes on, asking again as replay moves", ctx->promotion.label )); ctx->promotion.retry_logged = 1; } + /* Keep the bound final state and wait for replay before retrying. */ ctx->promotion.retry = 1; ctx->promotion.retry_slot = ctx->replay_slot; return; } if( FD_UNLIKELY( ctx->adopt_result.result!=FD_TOWER_ADOPT_SUCCESS ) ) { + /* Try the next permitted source before failing promotion. */ if( promote_fallback( ctx, adopt_err_name( ctx->adopt_result.result ) ) ) return; FD_LOG_WARNING(( "%s: the tower tile refused %s (%s)", ctx->promotion.label, source_name( ctx->promotion.source ), adopt_err_name( ctx->adopt_result.result ) )); ulong result = ctx->adopt_result.result; + /* Tower rejected the history, stop promotion and report any handoff refusal. */ reject_promotion( ctx, result==FD_TOWER_ADOPT_ERR_BLOCK_MISMATCH ? FD_FAILOVER_REJECT_ADOPTION_MISMATCH : FD_FAILOVER_REJECT_ADOPTION_FAILED, result==FD_TOWER_ADOPT_ERR_BLOCK_MISMATCH || result==FD_TOWER_ADOPT_ERR_DECODE || @@ -1400,14 +1433,17 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, return; } if( FD_UNLIKELY( ctx->promotion.adopt.tip!=FD_FAILOVER_SLOT_NULL && ctx->adopt_result.vote_slot!=ctx->promotion.adopt.tip ) ) { + /* Operator promotion may retry a different source after a tip mismatch. */ if( promote_fallback( ctx, "adoption did not retain its final vote" ) ) return; FD_LOG_WARNING(( "%s: the adopted tower ends at slot %lu, the tower says %lu, our root has passed its last vote, so it is a mismatch", ctx->promotion.label, ctx->adopt_result.vote_slot, ctx->promotion.adopt.tip )); + /* The adopted tip differs, fail promotion without installing the staked key. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_MISMATCH, 1 ); return; } int covered = floor_covered( ctx ); int empty_account = ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT && ctx->adopt_result.acct_vote_slot==FD_FAILOVER_SLOT_NULL; if( FD_UNLIKELY( !ctx->promotion.from_peer && ( !covered || empty_account ) ) ) { + /* Insufficient saved history falls back to the vote account. */ if( ctx->promotion.source!=FD_FAILOVER_SOURCE_VOTE_ACCOUNT && promote_fallback( ctx, "its votes do not cover the known signing history" ) ) return; if( !ctx->promotion.empty ) { @@ -1424,6 +1460,7 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, source_name( ctx->promotion.source ), ctx->adopt_result.vote_slot, ctx->promotion.floor )); ctx->promotion.floor_logged = 1; } + /* Stay in the adoption wait until the deadline expires. */ ctx->promotion.retry = 1; ctx->promotion.retry_slot = FD_FAILOVER_SLOT_NULL; return; @@ -1433,16 +1470,19 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, too. */ FD_LOG_WARNING(( "%s: another machine may hold the identity now, a peer authenticated as ACTIVE during this promotion, " "not installing the staked key, check `failover status` on both machines", ctx->promotion.label )); + /* Another holder was seen, stop before installing the staked key. */ reject_promotion( ctx, FD_FAILOVER_REJECT_HOLDS_IDENTITY, 0 ); return; } if( FD_UNLIKELY( ctx->promotion.adopt.tip==FD_FAILOVER_SLOT_NULL ) ) ctx->last_vote_slot = ctx->adopt_result.vote_slot; if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_STAKED )==ULONG_MAX ) ) { + /* The switch was not submitted, fail promotion while remaining standby. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } FD_LOG_NOTICE(( "%s: the tower tile adopted %s, asking the admin tile to install the staked identity", ctx->promotion.label, ctx->promotion.empty ? "an empty tower authorized by --force" : source_name( ctx->promotion.source ) )); + /* Adoption passed, wait for admin to install the staked identity. */ ctx->action = FD_FAILOVER_ACTION_PROMOTE_SWITCH; return; } @@ -1461,13 +1501,16 @@ step_promote_switch( fd_failover_tile_ctx_t * ctx ) { ctx->id_switch.fresh = 0; ctx->id_switch.overdue = 0; if( FD_UNLIKELY( ctx->id_switch.result.result!=FD_FAILOVER_SWITCH_OK ) ) { + /* Admin refused the switch, remain standby and fail the promotion. */ reject_promotion( ctx, FD_FAILOVER_REJECT_SWITCH_FAILED, 0 ); return; } + /* Admin confirmed the staked identity is installed, become active. */ set_role( ctx, FD_FAILOVER_ROLE_ACTIVE ); /* A new tenure starts, the peer tower we kept is older than its votes. */ ctx->peer_tower.valid = 0; ctx->taken = 0; + /* Local promotion is complete. */ ctx->action = FD_FAILOVER_ACTION_IDLE; ctx->stuck = 0; if( FD_UNLIKELY( ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ) ) { @@ -1480,6 +1523,7 @@ step_promote_switch( fd_failover_tile_ctx_t * ctx ) { /* The ACK tells the old active its handoff is done. We keep it as our response, so the same DEMOTED again gets the ACK again. */ ctx->promotion.from_peer = 0; + /* Remain active while the old active confirms our ACK. */ request_wait_result( ctx ); finish_reply( ctx, &ctx->promotion.peer, ctx->promotion.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_ACK, (uchar)FD_FAILOVER_REJECT_NONE ); } @@ -1497,7 +1541,7 @@ step_controller( fd_failover_tile_ctx_t * ctx, case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER: case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: - return; + return; /* Peer polling handles these waits. */ case FD_FAILOVER_ACTION_DEMOTE_SWITCH: step_demote_switch( ctx, stem ); @@ -1573,6 +1617,7 @@ request_pause( fd_failover_tile_ctx_t * ctx, long now, char const * why ) { FD_LOG_WARNING(( "handoff %lu paused (%s): %s, dialing stopped, %s", ctx->request.id, why, request_wait( ctx ), request_next( ctx ) )); + /* Pause dialing without changing the action or the authenticated binding. */ ctx->request.until = 0L; ctx->stuck = 1; fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); @@ -1619,6 +1664,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, return; } if( FD_UNLIKELY( request.handoff_id==ctx->handoff.id && peer_eq( &ctx->handoff.target, peer->boot_id, peer->junk_pubkey ) ) ) { + /* A duplicate request keeps the same demotion and reuses its eventual response. */ if( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH || ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN ) return; if( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) { if( !ctx->tx.valid ) (void)queue_demoted( ctx ); @@ -1636,6 +1682,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, promotion deadline. */ else if( request.replay_slot==FD_FAILOVER_SLOT_NULL || fd_ulong_sat_add( request.replay_slot, FD_FAILOVER_DEADLINE_SLOTS )last_vote_slot ) result = FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND; + /* Decline the request without changing the current role or action. */ if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { if( result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER && now>=ctx->handoff_refuse_log_at ) { ctx->handoff_refuse_log_at = fd_long_sat_add( now, 60000000000L ); @@ -1657,6 +1704,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, handoff_result( ctx, request.handoff_id, result ); return; } + /* An accepted request starts demotion on the active. */ start_demotion( ctx, 1 ); ctx->handoff.id = request.handoff_id; char member[ FD_BASE58_ENCODED_32_SZ ]; @@ -1687,6 +1735,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, char const * name = relayed_refusal( result.result, &hint ); FD_LOG_WARNING(( "handoff %lu ended (%s), %s", result.handoff_id, name, hint )); } + /* A bound final result ends the request without changing our installed identity. */ request_end( ctx, now, failed ? FD_FAILOVER_HANDOFF_DECLINED : FD_FAILOVER_HANDOFF_TAKEN ); if( !failed ) FD_LOG_NOTICE(( "handoff %lu complete: the old active confirmed our ACK, we hold the staked identity, on-demand connection closed", result.handoff_id )); @@ -1735,6 +1784,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, if( ctx->peer_floor==FD_FAILOVER_SLOT_NULL || tower.tip>ctx->peer_floor ) ctx->peer_floor = tower.tip; ctx->peer_tower = tower; ctx->session.peer_role = FD_FAILOVER_ROLE_STANDBY; + /* The bound final state moves the standby from peer wait to replay wait. */ start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &ctx->request.peer, demoted.handoff_id, 0 ); return; } @@ -1747,6 +1797,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, if( FD_UNLIKELY( !fd_failover_promote_ack_decode( &ack, ctx->rx, payload_sz ) ) ) break; if( FD_UNLIKELY( !demoted_answer_bound( ctx, ack.handoff_id ) ) ) return; FD_LOG_NOTICE(( "handoff %lu: received ACK, the peer took the staked identity, we stay a standby", ack.handoff_id )); + /* The peer ACK ends the demotion wait and records that it took the identity. */ handoff_resolved( ctx, FD_FAILOVER_HANDOFF_TAKEN ); ctx->taken = 1; ctx->session.peer_role = FD_FAILOVER_ROLE_ACTIVE; @@ -1772,6 +1823,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, "see the peer's log first", rej.handoff_id, reject_name( rej.reason ) )); } ctx->peer_tower.valid = 0; + /* The peer refusal ends the demotion wait, this machine stays standby. */ handoff_resolved( ctx, FD_FAILOVER_HANDOFF_DECLINED ); ctx->stuck = 1; handoff_result( ctx, rej.handoff_id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); @@ -1781,6 +1833,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, default: break; } + /* Malformed control traffic drops the connection and retains local work. */ fd_failover_channel_protocol_error( ctx->channel, now ); sync_session( ctx ); } @@ -1800,6 +1853,7 @@ request_end( fd_failover_tile_ctx_t * ctx, ctx->tx.valid = 0; ctx->session.close_after_send = 0; if( ctx->reply.state==FD_FAILOVER_REPLY_PENDING ) ctx->reply.state = FD_FAILOVER_REPLY_KEPT; + /* Return to idle after a response or cancellation. */ ctx->action = FD_FAILOVER_ACTION_IDLE; ctx->stuck = failed; fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); @@ -1816,12 +1870,14 @@ peer_poll( fd_failover_tile_ctx_t * ctx, if( FD_UNLIKELY( ctx->request.id && ctx->request.until && now>=ctx->request.until && ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) ) { + /* The network deadline pauses this request without cancelling it. */ request_pause( ctx, now, !ctx->request.peer.boot_id ? "no peer authenticated before the deadline" : !ctx->request.send_cnt ? "no handoff request sent before the deadline" : "no final answer before the deadline" ); return; } if( FD_UNLIKELY( ctx->session.close_after_send && !ctx->tx.valid && !fd_failover_channel_tx_pending( ctx->channel ) ) ) { + /* The result drained, close the session without changing role or action. */ ctx->session.close_after_send = 0; fd_failover_channel_hangup( ctx->channel, now ); FD_LOG_NOTICE(( "handoff %lu: confirmation drained, on-demand connection closed, listener ready for the next request", ctx->session.close_handoff_id )); @@ -1844,9 +1900,11 @@ peer_poll( fd_failover_tile_ctx_t * ctx, FD_LOG_WARNING(( "handoff %lu dialed a standby at `" FD_IP4_ADDR_FMT ":%hu`, give the active's address with `failover promote --address`, " "and if both machines show `role: standby` no machine holds the identity and `failover promote --force` on one of them takes it", ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), dial_port( ctx ) )); + /* End the peer wait without changing the installed identity. */ request_end( ctx, now, FD_FAILOVER_HANDOFF_NOT_ACTIVE ); return; } + /* Bind the request to the first authenticated active member and boot. */ ctx->request.peer.boot_id = peer->boot_id; fd_memcpy( ctx->request.peer.junk, peer->junk_pubkey, 32UL ); } else if( FD_UNLIKELY( !peer_eq( &ctx->request.peer, peer->boot_id, peer->junk_pubkey ) ) ) { @@ -1866,7 +1924,8 @@ peer_poll( fd_failover_tile_ctx_t * ctx, request_end( ctx, now, active ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ); return; } - /* The same member on a new boot will not demote for it. */ + /* A changed peer pauses the request without replacing its binding. + The same member on a new boot will not demote for it. */ ctx->request.peer_restarted = ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && fd_memeq( peer->junk_pubkey, ctx->request.peer.junk, 32UL ); request_pause( ctx, now, ctx->request.peer_restarted ? "the authenticated boot restarted" : "authenticated member or boot changed" ); return; @@ -1874,6 +1933,7 @@ peer_poll( fd_failover_tile_ctx_t * ctx, fd_failover_handoff_request_t request = { .handoff_id=ctx->request.id, .target_boot_id=ctx->request.peer.boot_id, .replay_slot=ctx->replay_slot }; if( FD_LIKELY( !fd_failover_channel_send( ctx->channel, now, FD_FAILOVER_MSG_HANDOFF_REQUEST, (uchar const *)&request, sizeof(request) ) ) ) { + /* The request is sent, keep waiting for final state or confirmation. */ ctx->request.sent = 1; ctx->request.send_cnt = fd_ulong_sat_add( ctx->request.send_cnt, 1UL ); if( now>=ctx->request.log_at ) { @@ -1946,12 +2006,14 @@ apply_control( fd_failover_tile_ctx_t * ctx, if( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->request.id && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ) { FD_LOG_NOTICE(( "handoff %lu: operator demote ends the wait for confirmation, giving up the staked identity locally", ctx->request.id )); + /* Operator demotion ends the confirmation wait before starting a local switch. */ request_end( ctx, now, FD_FAILOVER_HANDOFF_DECLINED ); } /* Drop the identity without handing it over. The peer is not asked or told anything, the other machine then needs fenced recovery. */ if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE ) ) return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_ACTIVE ) ) return FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; + /* An idle active starts a standalone demotion. */ start_demotion( ctx, 0 ); return FD_ADMINCTL_RESULT_SUCCESS; } @@ -1998,6 +2060,7 @@ apply_control( fd_failover_tile_ctx_t * ctx, ctx->request.until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); ctx->last_requested = 1; ctx->stuck = 0; + /* An idle standby starts a new handoff and waits for the active. */ ctx->action = FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER; ctx->session.close_after_send = 0; fd_failover_channel_init_dialer( ctx->channel, addr, dial_port( ctx ) ); @@ -2019,11 +2082,13 @@ apply_control( fd_failover_tile_ctx_t * ctx, ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ) { FD_LOG_NOTICE(( "handoff %lu cancelled by `failover promote --force` on the operator's word that the peer cannot sign, proceeding with local recovery", ctx->request.id )); + /* Forced recovery ends the peer wait without changing the installed identity. */ request_end( ctx, now, FD_FAILOVER_HANDOFF_CANCELLED ); } if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) { FD_LOG_WARNING(( "`failover promote --force` stops waiting for the peer's answer to handoff %lu, discarding saved final state because the peer may have voted, coverage floors retained", ctx->handoff.id )); ctx->peer_tower.valid = 0; + /* Forced recovery ends the ACK wait before starting local promotion. */ handoff_resolved( ctx, FD_FAILOVER_HANDOFF_CANCELLED ); } if( force ) FD_LOG_WARNING(( "operator promotion: --force asserts the peer cannot sign, peer checks are bypassed and incomplete or empty vote history is permitted if needed" )); @@ -2041,6 +2106,7 @@ apply_control( fd_failover_tile_ctx_t * ctx, fd_failover_tower_t const * saved = &ctx->peer_tower; FD_LOG_NOTICE(( "operator promotion: selecting %s through slot %lu, the tower tile must validate and adopt it before the staked identity is installed", source_name( source ), saved->tip )); } + /* The accepted operator command starts the replay and adoption sequence. */ start_promotion( ctx, source, NULL, 0UL, force ); return FD_ADMINCTL_RESULT_SUCCESS; } diff --git a/src/discof/failover/fd_failover_tls.c b/src/discof/failover/fd_failover_tls.c index 0777a60e731..ecd3d3b9be9 100644 --- a/src/discof/failover/fd_failover_tls.c +++ b/src/discof/failover/fd_failover_tls.c @@ -36,12 +36,14 @@ fd_failover_tls_ctx_init( fd_failover_tls_ctx_t * ctx, /* Both members present a certificate, the listener checks the dialer's. */ tls->req_client_cert = 1; + /* The TLS configuration is ready for new connections. */ ctx->ready = 1; return 0; } void fd_failover_tls_ctx_fini( fd_failover_tls_ctx_t * ctx ) { + /* Wiping the context prevents new connections until reinitialization. */ fd_memzero_explicit( ctx, sizeof(*ctx) ); } @@ -51,6 +53,7 @@ fd_failover_tls_new( fd_failover_tls_t * tls, int fd, int dial_peer ) { if( FD_UNLIKELY( !ctx->ready ) ) return -1; + /* Start an unverified, unpaired connection. */ fd_memset( tls, 0, sizeof(*tls) ); tls->fd = fd; fd_tlsrec_conn_init( &tls->conn, &ctx->tls, !dial_peer ); @@ -73,6 +76,7 @@ fd_failover_tls_fini( fd_failover_tls_t * tls ) { } fd_memzero_explicit( &tls->conn, sizeof(tls->conn) ); fd_memzero_explicit( &tls->sock, sizeof(tls->sock) ); + /* Closing clears both TLS verification and HELLO pairing. */ tls->fd = -1; tls->verified = 0; tls->paired = 0; @@ -107,7 +111,8 @@ pull( fd_failover_tls_t * tls ) { int fd_failover_tls_handshake( fd_failover_tls_t * tls ) { - if( FD_LIKELY( tls->verified ) ) return 1; + if( FD_LIKELY( tls->verified ) ) return 1; /* Verification is final for this connection. */ + /* A failed handshake stays failed until the channel closes it. */ if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) ) ) { FD_LOG_DEBUG(( "failover TLS handshake failed: %s", fd_tls_reason_cstr( tls->conn.hs.base.reason ) )); return -1; @@ -119,7 +124,7 @@ fd_failover_tls_handshake( fd_failover_tls_t * tls ) { FD_LOG_DEBUG(( "failover TLS handshake failed: %s", fd_tls_reason_cstr( tls->conn.hs.base.reason ) )); return -1; } - if( FD_LIKELY( !fd_tlsrec_conn_is_ready( &tls->conn ) ) ) return 0; + if( FD_LIKELY( !fd_tlsrec_conn_is_ready( &tls->conn ) ) ) return 0; /* Continue on a later poll. */ } /* Handshake done, the peer signed with some key and agreed on our ALPN. @@ -136,6 +141,7 @@ fd_failover_tls_handshake( fd_failover_tls_t * tls ) { fd_tls_estate_srv_t const * srv = &tls->conn.hs.srv; fd_memcpy( tls->peer_pubkey, srv->client_pubkey, 32UL ); } + /* TLS verified the peer key, HELLO must still authorize the member. */ tls->verified = 1; return 1; } From 94d9ad7c057f1e077b2f77160b40f9911c6c667f Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Thu, 8 Oct 2026 23:02:33 +0000 Subject: [PATCH 12/23] failover: make set-identity restart failover on the new identity and remove demote --- book/api/cli.md | 249 ++++++++++++++ book/snippets/commands/failover-promote.ansi | 5 + book/snippets/commands/failover-status.ansi | 13 + src/app/firedancer/commands/failover.c | 198 +++++++---- src/app/firedancer/commands/set_identity.c | 6 +- src/app/firedancer/config/default.toml | 18 +- src/app/firedancer/topology.c | 30 +- src/app/shared/commands/run/run.c | 19 +- src/app/shared/fd_action.h | 1 + src/app/shared/fd_config_json.c | 12 +- src/app/shared/test_config_json.c | 4 +- src/app/shared/test_config_parse.c | 1 + src/disco/events/generated/fd_event_gen.h | 3 +- src/disco/events/schema/admin_command.json | 3 +- src/disco/events/schema/events.proto | 3 +- src/disco/keyguard/fd_keyguard.h | 3 +- src/disco/keyguard/fd_keyswitch.h | 9 +- src/disco/keyguard/fd_sign_tile.c | 128 +++++-- src/disco/keyguard/test_sign_tile.c | 102 +++++- src/disco/topo/fd_topo.c | 6 +- src/discof/admin/fd_admin_tile.c | 293 +++++++++++++--- src/discof/admin/fd_adminctl.h | 48 +-- src/discof/admin/test_admin_tile.c | 182 ++++++++++ src/discof/failover/fd_failover_bus.h | 29 +- src/discof/failover/fd_failover_channel.c | 40 +-- src/discof/failover/fd_failover_channel.h | 5 +- src/discof/failover/fd_failover_tile.c | 336 +++++++++++-------- src/discof/failover/test_failover_tile.c | 175 +++++++++- 28 files changed, 1514 insertions(+), 407 deletions(-) create mode 100644 book/snippets/commands/failover-promote.ansi create mode 100644 book/snippets/commands/failover-status.ansi diff --git a/book/api/cli.md b/book/api/cli.md index bbaba117e07..882dc7696ab 100644 --- a/book/api/cli.md +++ b/book/api/cli.md @@ -219,6 +219,22 @@ the running validator will not be changed. Switching to the identity the validator is already running as is refused. +Between two Firedancer validators running failover, only use +`firedancer failover promote`, which hands over the vote history +directly. Do not use `--vote-history-file` for this, it is only for a +tower file from a validator that does not run failover, such as Agave. + +With failover on, `set-identity` also makes the new key the identity +`failover promote` hands off, unless it is the machine's junk identity. +Only a machine running the same key can take it over, so set that +key as `[paths.identity_key]` on the other machine and restart it. +After a restart, the machine boots as standby under its junk identity +and uses `[paths.identity_key]` as the handoff target again. +The new key cannot also be one of the authorized voters. With +`--vote-history-file`, a failover machine does not vote until the +file's root has been replayed, and ignores a file whose root is not on +the fork of its own root, waiting out the lockouts of its votes. + The validator writes `vote_history-.bin` in the `[paths.vote_history]` directory before each vote and each block it produces. This is an Agave-compatible vote history file and it protects @@ -333,6 +349,239 @@ than 16). <<< @/snippets/commands/add-authorized-voter.ansi +## `failover promote` +This command asks a remote active validator to hand off its identity and +vote history to the local running standby. Run this command on the +standby. The remote validator must have `[failover.enabled]` set to +`true` to accept the request. Both validators must use the same vote +account. + +The failover commands are only supported by the full Firedancer client. + +```sh +firedancer failover promote [] [--name ] [--yes] [--force] [--address
] [--port ] +``` + +The command attaches to the local running validator and discovers it +automatically. If more than one is running, pass `--name ` to +select one (see [`ps`](#ps)). It asks for confirmation when run from a +terminal unless `--yes` is given. Outside a terminal it proceeds +without a prompt. + +Before asking for confirmation, the command checks the local validator's +role. The connection to the remote validator starts after confirmation, +so the prompt does not mean the remote validator is reachable or ready +to hand off. + +The local validator finds the active's address through gossip. Use +`--address` to give the remote validator's IPv4 address or host name +instead, for example when connecting over a private network. Use +`--port` if its failover listener uses a different port from the local +validator's `[failover.listen_port]`. + +If gossip has no address for the remote validator, the command refuses +the request. Wait for its contact information or provide `--address`. +Providing an address does not bypass peer authentication. + +For example, to connect to a known failover listener over a private +network, run the following on the local standby. + +```sh +firedancer failover promote --name standby --address 192.0.2.10 --port 8010 +``` + +The remote validator must be reachable at that TCP address and port. If +it is stopped, its failover listener is disabled, or the connection +cannot be authenticated, the local request cannot complete the handoff. + +### Both validators have failover enabled + +Set `[failover.enabled]` to `true` on both validators. Configure the same +staked identity in `[paths.identity_key]` and the same vote account in +`[paths.vote_account]`. Firedancer creates a junk identity for each +validator if one does not already exist. Both boot under their junk +identities without voting and listen for handoff requests. + +Once one validator is active, run the following on the standby to request +a handoff. The staked identity keypair is already loaded from the +configuration, so no keypair argument is needed. + +```sh +firedancer failover promote --name standby +firedancer failover status --name standby +``` + +For example, an accepted request prints the handoff ID and the status +needed to confirm completion. + +<<< @/snippets/commands/failover-promote.ansi + +After the handoff, the old active can request the identity back with +`failover promote`. If both validators are still standbys, first promote +one with `--force` as described below. An optional keypair argument must +match the standby's current failover target. + +### Only the active has failover enabled + +Keep `[failover.enabled]` set to `true` on the active and set it to `false` +on the requester. Disabling failover does not disable voting. The +requester starts under `[paths.identity_key]` without generating a junk +identity or automatically becoming a nonvoting standby. + +For this workflow, start the requester with a junk identity in +`[paths.identity_key]` and the active's vote account in +`[paths.vote_account]`. The junk identity must differ from the active's +identity and must not be a staked node identity. This keeps the requester +nonvoting while it replays, even with authorized voter keys loaded. The +vote account must remain configured for takeover. + +If needed, create a junk key with the existing `keys` command and set +`[paths.identity_key]` to its path before starting the requester. + +```sh +firedancer keys new /path/to/standby-identity.json +firedancer run --config /path/to/standby.toml +``` + +Validators with a vote account include a failover tile for outgoing +requests, so allow for it when assigning CPU affinity. Pass the active's +identity keypair on first use. This prepares the target without installing +it. The requester must still be using the identity it started with. The +target cannot also be an authorized voter. + +```sh +firedancer failover promote /path/to/active-identity.json --name standby +firedancer failover status --name standby +``` + +Later requests can omit the keypair. Repeating the same keypair preserves +an ongoing handoff. A different keypair is refused. After takeover, the +requester's failover listener stays disabled, so the other validator +cannot request the identity back from it. On restart, it returns to its +configured identity and forgets the prepared target, so the next request +needs the keypair again. + +If both validators have failover disabled, neither can accept a handoff +request. + +### Handoff progress and failures + +The following applies to both configurations. + +The remote validator refuses the handoff if it has no final vote history +to hand over, the local standby has not started replaying, or the standby +is more than 64 slots behind its last vote. It keeps its identity in +these cases. It also refuses a new request while another transition is +running. Once it agrees to hand off, it gives up the identity before the +local validator adopts its vote history. If adoption fails or stalls, +the handoff can fail with neither machine voting. + +Exit code 0 means the local validator accepted the request. It does not +mean the remote validator has agreed to hand off or that the handoff has +completed. The command prints a handoff ID. Check +[`failover status`](#failover-status) on the local validator for +`role: active` and `last handoff: , taken` with that ID. Check that the +remote validator shows `role: standby` and `action: idle`. + +The local validator refuses a new request if it is already active or +another transition is running. A local refusal exits with a non-zero +code and prints the reason to `stderr`. An accepted request can still +fail while connecting to the remote validator, authenticating it, or +adopting its vote history. These later failures do not change the +command's exit code. Check status and the logs on both validators to +determine where it stopped. + +If the command itself times out waiting for the local validator to +answer, the outcome is unknown. Check status on both machines because +the request may still complete. + +If the request pauses at its network deadline, the local validator stops +dialing. Run `failover status` on both validators before retrying. Run +`failover promote` again on the local validator to resume the +request. This also resumes an active validator that is waiting for the +remote validator to confirm completion. You can provide a new `--address` +or `--port` when resuming it. + +### Forced promotion + +With `--force`, the local standby takes the identity without contacting +the remote validator. This is also needed for first use when both +machines are standbys. Promotion first tries eligible saved peer vote +history, then the vote account, and starts with empty vote history if +neither can be adopted. Check for `role: active` and `action: idle` to +confirm completion. + +Forced promotion can cancel a handoff that is waiting for the remote +validator's response. It cannot interrupt a local vote history adoption +or identity switch. Replay and adoption must still complete before the +identity can be installed. + +::: warning WARNING + +Only use `--force` after ensuring every other machine with the identity +cannot sign. It skips the checks on the other machine and can discard +earlier vote lockouts by accepting incomplete or empty vote history. +Never force promotion on both machines at once. `--yes` only skips the +confirmation prompt. It does not imply `--force` or suppress its warning. + +::: + +| Arguments | Description | +|-----------------------|-------------| +| `` | Optionally provide the target identity keypair file. Required on first use when `[failover.enabled]` is `false`. | +| `--name ` | Select the validator instance to attach to if more than one is running on this host. | +| `--config ` | Optionally use the configuration TOML file the validator was started with. | +| `--yes` | Skip the confirmation prompt. | +| `--force` | Take the identity without cooperation from the remote validator, allowing incomplete or empty vote history. This option cannot be combined with `--address` or `--port`. | +| `--address
` | Use this IPv4 address or host name for the remote validator instead of the address found through gossip. This option cannot be combined with `--force`. | +| `--port ` | Set the remote validator's failover TCP port. The default is the local validator's `[failover.listen_port]`, which is 8010 unless configured otherwise. This option cannot be combined with `--force`. | + +## `failover status` +This command prints the local running validator's failover role, current +action, peer and last handoff. It also reports what this machine knows +about another holder of the identity and the vote history a forced +promotion would adopt. Use it to check progress after +[`failover promote`](#failover-promote). + +```sh +firedancer failover status [--name ] +``` + +The command discovers the local running validator automatically. If more +than one is running, pass `--name ` to select one (see [`ps`](#ps)). +It exits with code 0 if status was retrieved, including when failover +is disabled. Otherwise it exits with a non-zero code and prints the +reason to `stderr`. + +With `[failover.enabled]` set to `false` and a vote account configured, +status reports `client only, listener disabled` and shows outgoing +handoff progress. + +The following example shows a completed handoff. The last handoff ID +matches the ID printed by `failover promote` above. The connection closes +after completion, so `peer role` is unknown until another authenticated +session is established. + +<<< @/snippets/commands/failover-status.ansi + +`role` describes the local validator. `peer role` describes the remote +validator only while an authenticated session is open. `last handoff` +records the latest request's outcome, so check `role` and `action` as +well when investigating a failed or paused handoff. + +`other holder` reports what the local validator knows about another +machine using the identity. Seeing none does not prove every other +machine is unable to sign. `takeover adopts` shows the vote history +source a forced promotion would currently select. `coverage floor` is +the highest slot known to have been signed locally or reported by the +remote validator. Forced promotion can proceed without history covering +that slot, as described above. + +| Arguments | Description | +|-------------------|-------------| +| `--name ` | Select the validator instance to attach to if more than one is running on this host. | +| `--config ` | Optionally use the configuration TOML file the validator was started with. | + ## `remove-all-authorized-voters` Removes all authorized voters from the running validator, including any seeded from `[paths.authorized_voter_paths]` at startup as well as any diff --git a/book/snippets/commands/failover-promote.ansi b/book/snippets/commands/failover-promote.ansi new file mode 100644 index 00000000000..2272ba232fb --- /dev/null +++ b/book/snippets/commands/failover-promote.ansi @@ -0,0 +1,5 @@ +$ firedancer failover promote --name standby --yes +failover promote accepted +role: standby +action: handoff, requesting the active's final vote state +handoff: 123456789, done when `failover status` here shows role: active and last handoff: 123456789, taken diff --git a/book/snippets/commands/failover-status.ansi b/book/snippets/commands/failover-status.ansi new file mode 100644 index 00000000000..77b54553d0d --- /dev/null +++ b/book/snippets/commands/failover-status.ansi @@ -0,0 +1,13 @@ +$ firedancer failover status --name standby +failover: enabled +role: active +action: idle +stuck: no +link: listening +peer role: unknown, no authenticated failover session +peer boot id: 0000000000000001 +peer address: 192.0.2.10:8010, from gossip +last handoff: 123456789, taken +other holder: n/a, this machine holds the identity +takeover adopts: the vote account (automatic fallback) +coverage floor: slot 300 diff --git a/src/app/firedancer/commands/failover.c b/src/app/firedancer/commands/failover.c index 5ae7bd39e6f..25225fbb1bf 100644 --- a/src/app/firedancer/commands/failover.c +++ b/src/app/firedancer/commands/failover.c @@ -5,17 +5,20 @@ #include "../../../discof/failover/fd_failover_proto.h" #include "../../../disco/topo/fd_dns_resolve.h" #include "../../../util/net/fd_ip4.h" +#include "../../../disco/keyguard/fd_keyload.h" +#include "../../platform/fd_cap_chk.h" #include #include #include #include +#include static char const * const ACTION_NAMES[] = { - "idle", "demote, switching to the junk key", "demote, waiting for the peer's answer", + "idle", "handing off, switching to the junk key", "handing off, waiting for the peer's answer", "promote, waiting for replay", "promote, waiting for vote history adoption", "promote, switching to the staked key", "handoff, requesting the active's final vote state", "handoff, waiting for completion", - "demote, waiting for the tower to drain" + "handing off, waiting for the tower to drain" }; static char const * const SESSION_NAMES[] = { "listening", "dialing", "hello", "paired", "backoff" @@ -56,7 +59,8 @@ strip_value( int * pargc, static void failover_cmd_args( int * pargc, char *** pargv, - args_t * args ) { + args_t * args, + int status ) { char const * name = strip_value( pargc, pargv, "--name" ); char const * address = strip_value( pargc, pargv, "--address" ); char const * port = strip_value( pargc, pargv, "--port" ); @@ -64,21 +68,13 @@ failover_cmd_args( int * pargc, args->failover.yes = fd_env_strip_cmdline_contains( pargc, pargv, "--yes" ); args->failover.force = fd_env_strip_cmdline_contains( pargc, pargv, "--force" ); - if( FD_UNLIKELY( !( *pargc ) ) ) { - FD_LOG_ERR(( "missing subcommand, supported: status, promote, demote" )); - } - /* `promote --force` bypasses checks and takes the identity immediately - without a clean hand-off. */ - char const * cmd = **pargv; - if( !strcmp( cmd, "status" ) ) args->failover.cmd = (int)FD_ADMINCTL_FAILOVER_CMD_STATUS; - else if( !strcmp( cmd, "demote" ) ) args->failover.cmd = (int)FD_ADMINCTL_FAILOVER_CMD_DEMOTE; - else if( !strcmp( cmd, "promote" ) ) args->failover.cmd = (int)( args->failover.force ? FD_ADMINCTL_FAILOVER_CMD_PROMOTE : FD_ADMINCTL_FAILOVER_CMD_HANDOFF ); - else FD_LOG_ERR(( "unknown subcommand `%s`, supported: status, promote, demote", cmd )); - ( *pargc )--; - ( *pargv )++; + /* `failover promote --force` bypasses checks and takes the identity immediately + without a clean handoff. */ + args->failover.cmd = (int)( status ? FD_ADMINCTL_FAILOVER_CMD_STATUS : + args->failover.force ? FD_ADMINCTL_FAILOVER_CMD_PROMOTE : FD_ADMINCTL_FAILOVER_CMD_HANDOFF ); if( FD_UNLIKELY( args->failover.yes && args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_STATUS ) ) { - FD_LOG_ERR(( "--yes is only meaningful for `failover promote` and `failover demote`" )); + FD_LOG_ERR(( "--yes is only meaningful for `failover promote`" )); } if( FD_UNLIKELY( args->failover.force && args->failover.cmd!=(int)FD_ADMINCTL_FAILOVER_CMD_PROMOTE ) ) { FD_LOG_ERR(( "--force is only meaningful for `failover promote`" )); @@ -106,6 +102,24 @@ failover_cmd_args( int * pargc, } args->failover.port = (ushort)value; } + if( !status && *pargc && (*pargv)[ 0 ][ 0 ]!='-' ) { + fd_cstr_ncpy( args->failover.keypair_path, (*pargv)[ 0 ], sizeof(args->failover.keypair_path) ); + (*pargc)--; + (*pargv)++; + } +} + +static void +failover_args( int * pargc, + char *** pargv, + args_t * args ) { + if( FD_UNLIKELY( !*pargc || ( strcmp( (*pargv)[ 0 ], "promote" ) && strcmp( (*pargv)[ 0 ], "status" ) ) ) ) { + FD_LOG_ERR(( "usage: firedancer failover [OPTIONS]" )); + } + int status = !strcmp( (*pargv)[ 0 ], "status" ); + (*pargc)--; + (*pargv)++; + failover_cmd_args( pargc, pargv, args, status ); } static char const * @@ -125,7 +139,7 @@ action_name( uchar action ) { static char const * control_result_name( ulong result ) { switch( result ) { - case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: return "this machine is not in the role that command needs, `promote` runs on a standby, `demote` on the active"; + case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: return "this machine is not in the role that command needs, `failover promote` runs on a standby"; case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: return "a transition or key switch is running, check `failover status`"; case FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS: return "gossip has no address for the active, wait or give it with --address, and if both machines show " "`role: standby` no machine holds the identity and `failover promote --force` on one of them takes it"; @@ -138,15 +152,20 @@ control_result_name( ulong result ) { case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "gossip showed the staked identity at another host within the last 15 seconds, an active is publishing"; case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER: return "the active has no eligible final vote state to hand over and keeps the identity, check its voting progress and vote-history logs before retrying"; case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: return "this standby is too far behind the active's last vote and the active keeps the identity, run `failover promote` again once this machine has caught up"; + case FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED: return "provide the active's keypair with `failover promote ` to prepare this client's target identity"; + case FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT: return "the keypair must match the prepared target and must differ from this machine's boot identity"; + case FD_FAILOVER_CONTROL_RESULT_NOT_BOOT_IDENTITY: return "the client must still be using its boot identity when preparing a target"; + case FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER: return "the target identity is already an authorized voter, remove it from the authorized voters before preparing it"; + case FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH: return "the public key in the keypair file does not match its private key"; default: return NULL; } } static void failover_confirm( args_t const * args ) { - /* Each of these moves or drops the staked identity, so the operator - confirms first. What `failover promote --force` rests on is printed - even with --yes. */ + /* A promote hands off the staked identity to this machine, so the operator + confirms first. The warning for `failover promote --force` is + printed even with --yes. */ if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_PROMOTE ) { FD_LOG_STDOUT(( "This takes the staked identity on your word. No other machine may hold it or be\n" "in a promotion, including any validator running this identity without failover.\n" @@ -160,18 +179,13 @@ failover_confirm( args_t const * args ) { "Promotion tries eligible saved vote state first, then the vote account, and if the\n" "vote account cannot be adopted it starts from an EMPTY vote history.\n" )); } - if( FD_UNLIKELY( !args->failover.yes ) ) { + if( FD_UNLIKELY( !args->failover.yes && isatty( STDIN_FILENO ) ) ) { if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) { - FD_LOG_STDOUT(( "This asks the active to hand the staked identity to this standby. The active\n" + FD_LOG_STDOUT(( "This asks the active to hand off the staked identity to this standby. The active\n" "keeps it if this standby is more than 64 slots behind its last vote. Otherwise\n" "it gives up the identity first, so if replay here stalls before it catches up\n" "the handoff fails and neither machine votes until `failover promote --force`.\n" "Check that this machine is caught up. Type yes to continue: " )); - } else if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_DEMOTE ) { - FD_LOG_STDOUT(( "This gives up the staked identity and tells the other machine nothing. Nothing\n" - "votes until `failover promote --force` runs on one machine. To move the\n" - "identity to the other machine, run `failover promote` there instead. Type yes\n" - "to continue: " )); } else { FD_LOG_STDOUT(( "Type yes to continue: " )); } @@ -187,6 +201,7 @@ failover_confirm( args_t const * args ) { static char const * other_holder( ulong result ) { switch( result ) { + case FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED: return "no target identity prepared"; case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: return "n/a, this machine holds the identity"; case FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE: return "the authenticated peer says it holds the identity"; case FD_FAILOVER_CONTROL_RESULT_TAKEN: return "the peer took our last handoff and may be voting"; @@ -201,7 +216,7 @@ other_holder( ulong result ) { } } -/* Prints failover status. */ +/* Prints the response to `failover status`. */ static void failover_status_print( fd_adminctl_failover_status_resp_t const * resp ) { if( FD_UNLIKELY( !resp->enabled ) ) { @@ -209,10 +224,10 @@ failover_status_print( fd_adminctl_failover_status_resp_t const * resp ) { return; } - FD_LOG_STDOUT(( "%-22s %s\n", "failover:", "enabled" )); + FD_LOG_STDOUT(( "%-22s %s\n", "failover:", resp->enabled==2 ? "client only, listener disabled" : "enabled" )); FD_LOG_STDOUT(( "%-22s %s\n", "role:", role_name ( resp->role ) )); if( FD_UNLIKELY( resp->request_paused ) ) { - FD_LOG_STDOUT(( "%-22s %s, paused at its deadline and not dialing, `failover promote` resumes it, with --yes on the active\n", "action:", action_name( resp->action ) )); + FD_LOG_STDOUT(( "%-22s %s, paused at its deadline and not dialing, `failover promote` resumes it\n", "action:", action_name( resp->action ) )); } else { FD_LOG_STDOUT(( "%-22s %s\n", "action:", action_name( resp->action ) )); } @@ -241,7 +256,7 @@ failover_status_print( fd_adminctl_failover_status_resp_t const * resp ) { FD_LOG_STDOUT(( "%-22s %s\n", "takeover adopts:", resp->promote_sourcepromote_source ] : "unknown" )); if( FD_UNLIKELY( resp->promote_floor==FD_FAILOVER_SLOT_NULL ) ) FD_LOG_STDOUT(( "%-22s none known\n", "coverage floor:" )); - else FD_LOG_STDOUT(( "%-22s slot %lu\n", "coverage floor:", resp->promote_floor )); + else FD_LOG_STDOUT(( "%-22s slot %lu\n", "coverage floor:", resp->promote_floor )); } /* Signal handler that tells the operator a command is still waiting. */ @@ -256,9 +271,10 @@ wait_note( int sig ) { /* Sends req to the failover tile and waits for the answer, which lands in resp. Returns the adminctl result. */ -static ulong +static ulong FD_FN_SENSITIVE failover_request( fd_adminctl_t * adminctl, fd_adminctl_failover_req_t const * req, + char const * keypair_path, void * resp, ulong resp_max, ulong * resp_sz ) { @@ -266,9 +282,16 @@ failover_request( fd_adminctl_t * adminctl, ulong payload_max = 0UL; ulong slot_idx = fd_adminctl_reserve( adminctl, &payload, &payload_max ); if( FD_UNLIKELY( slot_idx==ULONG_MAX ) ) FD_LOG_ERR(( "all admin command slots are busy" )); - if( FD_UNLIKELY( sizeof(*req)>payload_max ) ) FD_LOG_ERR(( "adminctl failover payload too large" )); + ulong payload_sz = keypair_path ? sizeof(fd_adminctl_failover_prepare_t) : sizeof(*req); + if( FD_UNLIKELY( payload_sz>payload_max ) ) FD_LOG_ERR(( "adminctl failover payload too large" )); fd_memcpy( payload, req, sizeof(*req) ); - fd_adminctl_publish( adminctl, slot_idx, FD_ADMINCTL_CMD_FAILOVER, sizeof(*req) ); + if( keypair_path ) { + uchar const * keypair = fd_keyload_load( keypair_path, 0 ); + fd_memcpy( ((fd_adminctl_failover_prepare_t *)payload)->keypair, keypair, 64UL ); + fd_keyload_unload( keypair, 0 ); + } + ulong cmd = req->cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? FD_ADMINCTL_CMD_FAILOVER_STATUS : FD_ADMINCTL_CMD_FAILOVER_PROMOTE; + fd_adminctl_publish( adminctl, slot_idx, cmd, payload_sz ); struct sigaction sa = { .sa_handler = wait_note }; if( FD_UNLIKELY( sigaction( SIGALRM, &sa, NULL ) ) ) FD_LOG_ERR(( "sigaction(SIGALRM) failed (%i-%s)", errno, fd_io_strerror( errno ) )); @@ -280,8 +303,8 @@ failover_request( fd_adminctl_t * adminctl, static void __attribute__((noreturn)) fail_disabled( int status ) { - if( status ) FD_LOG_ERR(( "this validator has no failover command bus, so it cannot report failover status" )); - FD_LOG_ERR(( "failover is not enabled on the running validator, set [failover.enabled] to true and restart it" )); + if( status ) FD_LOG_ERR(( "`failover status` is unavailable because this validator has no failover command bus" )); + FD_LOG_ERR(( "the running validator has no failover client, use a supported Firedancer build with [paths.vote_account] configured" )); } static void __attribute__((noreturn)) @@ -295,17 +318,18 @@ fail_incompatible( void ) { static void failover_precheck( fd_adminctl_t * adminctl, args_t const * args ) { - fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_STATUS }; + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_STATUS }; fd_adminctl_failover_status_resp_t status; - ulong status_sz = 0UL; - ulong result = failover_request( adminctl, &req, &status, sizeof(status), &status_sz ); + ulong status_sz = 0UL; + ulong result = failover_request( adminctl, &req, NULL, &status, sizeof(status), &status_sz ); if( FD_UNLIKELY( result==FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ) ) fail_disabled( 0 ); if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS || status_sz!=sizeof(status) || status.version!=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION ) ) return; - if( FD_UNLIKELY( status.enabled!=1 ) ) fail_disabled( 0 ); - int want_active = args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_DEMOTE; - if( FD_UNLIKELY( (status.role==FD_FAILOVER_ROLE_ACTIVE)!=want_active ) ) { - FD_LOG_ERR(( "`failover %s` refused, %s", fd_adminctl_failover_cmd_typed( (ulong)args->failover.cmd ), + if( FD_UNLIKELY( !status.enabled ) ) fail_disabled( 0 ); + if( FD_UNLIKELY( status.role==FD_FAILOVER_ROLE_ACTIVE && + !( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF && + status.request_paused && status.action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) ) { + FD_LOG_ERR(( "`%s` refused, %s", fd_adminctl_failover_cmd_typed( (ulong)args->failover.cmd ), control_result_name( FD_FAILOVER_CONTROL_RESULT_BAD_ROLE ) )); } } @@ -316,7 +340,7 @@ failover_cmd_fn( args_t * args, config_t * config ) { fd_adminctl_t * adminctl = adminctl_client_attach( config, args->failover.name ); int status = args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_STATUS; - if( !status && !args->failover.yes ) failover_precheck( adminctl, args ); + if( !status && !args->failover.yes && isatty( STDIN_FILENO ) ) failover_precheck( adminctl, args ); if( !status ) failover_confirm( args ); /* --yes only skips the prompt, the operator still has to give --force @@ -326,7 +350,7 @@ failover_cmd_fn( args_t * args, req.version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION; req.cmd = (ulong)args->failover.cmd; if( FD_UNLIKELY( args->failover.yes ) ) req.flags |= FD_ADMINCTL_FAILOVER_FLAG_YES; - if( FD_UNLIKELY( args->failover.force ) ) req.flags |= FD_ADMINCTL_FAILOVER_FLAG_FORCE; + if( FD_UNLIKELY( args->failover.force ) ) req.flags |= FD_ADMINCTL_FAILOVER_FLAG_FORCE; req.addr = args->failover.addr; req.port = args->failover.port; @@ -335,10 +359,11 @@ failover_cmd_fn( args_t * args, fd_adminctl_failover_status_resp_t status; } resp = { 0 }; ulong resp_sz = 0UL; - ulong result = failover_request( adminctl, &req, &resp, sizeof(resp), &resp_sz ); + char const * keypair = args->failover.keypair_path[ 0 ] ? args->failover.keypair_path : NULL; + ulong result = failover_request( adminctl, &req, keypair, &resp, sizeof(resp), &resp_sz ); char const * cmd_name = fd_adminctl_failover_cmd_typed( (ulong)args->failover.cmd ); char const * refusal = control_result_name( result ); - if( FD_UNLIKELY( refusal ) ) FD_LOG_ERR(( "`failover %s` refused, %s", cmd_name, refusal )); + if( FD_UNLIKELY( refusal ) ) FD_LOG_ERR(( "`%s` refused, %s", cmd_name, refusal )); switch( result ) { case FD_ADMINCTL_RESULT_SUCCESS: if( FD_UNLIKELY( resp_sz!=(status ? sizeof(resp.status) : sizeof(resp.control)) || @@ -355,8 +380,6 @@ failover_cmd_fn( args_t * args, FD_LOG_STDOUT(( "%-22s %s\n", "action:", action_name( resp.control.action ) )); if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) { FD_LOG_STDOUT(( "%-22s %lu, done when `failover status` here shows role: active and last handoff: %lu, taken\n", "handoff:", id, id )); - } else if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_DEMOTE ) { - FD_LOG_STDOUT(( "%-22s %lu, done when `failover status` here shows role: standby and action: idle\n", "demotion:", id )); } else { FD_LOG_STDOUT(( "%-22s done when `failover status` here shows role: active and action: idle\n", "promotion:" )); } @@ -383,44 +406,71 @@ failover_cmd_fn( args_t * args, } } +static void +failover_status_args_help( fd_action_help_t * help ) { + fd_action_help_arg( help, "--name", "", "Name of the validator instance to attach to, if more than one is\n" + "running on this host" ); +} + static void failover_args_help( fd_action_help_t * help ) { - fd_action_help_arg( help, "--name", "", "Name of the validator instance to attach to, if more than one is\n" - "running on this host" ); - fd_action_help_arg( help, "--yes", NULL, "Skip the confirmation prompt. It does not authorize peer or\n" - "vote-history overrides" ); - fd_action_help_arg( help, "--force", NULL, "Only with `promote`. Take the identity without the active, skip\n" - "every check on the other machine and stop waiting for its answer\n" - "to our handoff. Use it only when the other machine cannot sign.\n" - "Also accept incomplete or empty vote history, earlier lockouts may\n" - "be lost" ); - fd_action_help_arg( help, "--address", "
", "Only with `promote` without --force. The active to ask, an IPv4\n" - "address or host name, instead of the address gossip shows for it,\n" - "for example when the active listens on a private network" ); - fd_action_help_arg( help, "--port", "", "Only with `promote` without --force. The active's failover port.\n" - "Without it we dial our own [failover.listen_port], so both machines\n" - "normally use the same port" ); + fd_action_help_arg( help, "", NULL, "Request takeover on this validator, or read its failover state" ); + fd_action_help_arg( help, "[]", NULL, "Path to the target identity keypair. Required on first use when\n" + "[failover.enabled] is false. Loading it does not install it" ); + failover_status_args_help( help ); + fd_action_help_arg( help, "--yes", NULL, "Skip the confirmation prompt. It does not authorize peer or\n" + "vote-history overrides" ); + fd_action_help_arg( help, "--force", NULL, "Take the identity without the active, skip\n" + "every check on the other machine and stop waiting for its answer\n" + "to our handoff. Use it only when the other machine cannot sign.\n" + "Also accept incomplete or empty vote history, earlier lockouts may\n" + "be lost" ); + fd_action_help_arg( help, "--address", "
", "Only without --force. The active to ask, an IPv4\n" + "address or host name, instead of the address gossip shows for it,\n" + "for example when the active listens on a private network" ); + fd_action_help_arg( help, "--port", "", "Only without --force. The active's failover port.\n" + "Without it we dial our own [failover.listen_port], so both machines\n" + "normally use the same port" ); +} + +static void +failover_promote_perm( args_t * args, + fd_cap_chk_t * chk, + config_t const * config FD_PARAM_UNUSED ) { + if( args->failover.keypair_path[ 0 ] ) { + fd_cap_chk_raise_rlimit( chk, "failover promote", RLIMIT_MEMLOCK, 5UL*FD_SHMEM_NORMAL_PAGE_SZ, + "lock the target keypair in memory" ); + } } action_t fd_action_failover = { .name = "failover", - .args = failover_cmd_args, + .args = failover_args, .fn = failover_cmd_fn, .require_config = 0, - .perm = NULL, - .description = "Inspect and drive failover of the running validator", + .perm = failover_promote_perm, + .description = "Promote the running standby validator or read its failover state", .detail = "Hands off the staked identity between two validators. A validator with\n" "[failover] enabled boots as a standby under its junk identity and does\n" - "not vote until promoted.\n" + "not vote until promoted. Both validators must use the same\n" + "[paths.vote_account].\n" + "With [failover.enabled] false, start the requester with an unstaked junk\n" + "key in [paths.identity_key] to keep it nonvoting until takeover. Provide\n" + "the active's keypair on first use. The requester keeps its boot identity\n" + "until adoption and the identity switch complete. It does not listen for\n" + "incoming handoffs. The target must differ from the boot identity and\n" + "cannot be an authorized voter.\n" "\n" - "`status` prints this machine's failover state. `promote` on the standby\n" - "asks the active to hand over its identity. `promote --force` takes the\n" - "identity without the active, for when it cannot sign. Each asks for\n" - "confirmation unless --yes is given.\n" + "Asks the active to hand over its identity. With --force, takes the\n" + "identity without the active, for when it cannot sign. Both forms\n" + "ask for confirmation in a terminal unless --yes is given.\n" + "Outside a terminal they proceed without a prompt. Use `failover status`\n" + "to check progress.\n" "\n" - "This command does not start a validator; it attaches to one that is already\n" + "This command does not start a validator. It attaches to one that is already\n" "running. With no arguments it discovers the running validator automatically.\n" "If multiple validators are running, pass --name to select one.\n", - .usage = "failover status|promote|demote [--name ] [--yes] [--force] [--address
] [--port ]", + .usage = "failover promote [] [--name ] [--yes] [--force] [--address
] [--port ]\n" + " firedancer failover status [--name ]", .args_help = failover_args_help, }; diff --git a/src/app/firedancer/commands/set_identity.c b/src/app/firedancer/commands/set_identity.c index ebcdf78c316..358c5681e22 100644 --- a/src/app/firedancer/commands/set_identity.c +++ b/src/app/firedancer/commands/set_identity.c @@ -132,9 +132,9 @@ set_identity( args_t * args, case FD_SET_IDENTITY_RESULT_INVALID_VOTE_HISTORY: FD_LOG_ERR(( "Failed to set identity: the vote history file is invalid or does not belong to the new identity. " "If you believe the vote history file is valid, please contact the Firedancer team for additional assistance." )); - case FD_ADMINCTL_RESULT_UNSUPPORTED: - FD_LOG_ERR(( "Failed to set identity: the validator runs with [failover.enabled] set to true, " - "and failover fully manages the identity while it is on." )); + case FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER: + FD_LOG_ERR(( "Failed to set identity: the new failover identity is already an authorized voter. " + "Remove it from the authorized voters before selecting it as the identity." )); case FD_ADMINCTL_RESULT_UNKNOWN_COMMAND: case FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH: case FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH: diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index e486e15ce29..4971afac388 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -1974,9 +1974,10 @@ telemetry = true # loading. delay_startup = true -# Firedancer validators support a special failover protocol between one -# another, which guarantees fast, safe, at-most-once handover with no -# equivocation or voting violations. +# Firedancer validators support a coordinated failover protocol. A +# handoff transfers the final vote history after the active stops signing. +# `failover promote --force` bypasses peer checks and can discard earlier +# lockouts or allow both validators to vote. # # The protocol works by hosting a TCP listener on your validator. Any # peer in possession of the currently active identity private key can @@ -1990,9 +1991,14 @@ telemetry = true # history files are only for handing off an identity from a validator that # does not run failover, such as Agave. [failover] - # Whether to enable the failover protocol on this validator, which - # exposes the TCP listener and allows peers to securely demote it to - # the junk identity. + # Whether to accept incoming handoff requests. When true, the + # validator boots as standby under its junk identity. + # When false, it boots normally under [paths.identity_key]. For a + # nonvoting requester, use an unstaked junk key as [paths.identity_key] + # and the active's vote account as [paths.vote_account]. Request an + # outgoing handoff using `failover promote `. The target + # keypair is prepared without installing it and must differ from the + # boot identity. enabled = false # The address to listen on. If enabled, by default the listener is diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index e5fbacc1fea..f146e3c3dec 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -285,7 +285,7 @@ fd_topo_initialize( config_t * config ) { int leader_enabled = !!config->firedancer.layout.enable_block_production; int rserve_enabled = config->tiles.rserve.enabled; int efficient_mode = !strcmp( config->firedancer.layout.mode, "efficient" ); - int failover_enabled = config->firedancer.failover.enabled; + int failover_enabled = config->firedancer.failover.enabled || config->paths.vote_account[ 0 ]; if( FD_UNLIKELY( snapmk_enabled ) ) { FD_CHECK_ERR( config->firedancer.snapshots.max_full_snapshots_to_keep, @@ -783,14 +783,14 @@ fd_topo_initialize( config_t * config ) { if( !config->tiles.gui.enabled ) { fd_topob_tile_out( topo, "gossip", 0UL, "gossip_ciseen", 0UL ); } - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "gossip_ciseen", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* contact infos for the peer's address, an overrun only delays it */ + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "gossip_ciseen", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /* contact infos for the peer's address */ /**/ fd_topob_tile_out( topo, "admin", 0UL, "admin_failov", 0UL ); /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "admin_failov", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_admin", 0UL ); /**/ fd_topob_tile_in ( topo, "admin", 0UL, "metric_in", "failov_admin", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); } if( !alpenglow_enabled && failover_enabled ) { - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "failov", 0UL, "adopt_tower", 0UL ); /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_adopt", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); } @@ -943,10 +943,10 @@ fd_topo_initialize( config_t * config ) { } if( FD_UNLIKELY( failover_enabled ) ) { - /**/ fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "failov_sign", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); - /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_sign", 0UL ); - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "sign_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED ); - /**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_failov", 0UL ); + /**/ fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "failov_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_sign", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "sign_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED ); + /**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_failov", 0UL ); } if( FD_UNLIKELY( rpc_enabled ) ) { @@ -1425,8 +1425,8 @@ wfs_decode_bank_hash( fd_config_t const * config, void fd_topo_configure_tile( fd_topo_tile_t * tile, fd_config_t * config ) { - /* Failover machines boot under the junk identity, only promotion - installs the staked key. A demoted machine rebooting under the + /* Failover machines boot under the junk identity, only a promotion or + set-identity installs a voting key. A demoted machine rebooting under the staked key would make the doppelganger rule shut down the promoted peer instead. */ char const * identity_key_path = config->firedancer.failover.enabled @@ -1508,10 +1508,10 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "failov" ) ) ) { - fd_cstr_ncpy( tile->failov.identity_key_path, identity_key_path, sizeof(tile->failov.identity_key_path) ); - fd_cstr_ncpy( tile->failov.staked_key_path, config->paths.identity_key, sizeof(tile->failov.staked_key_path) ); - fd_cstr_ncpy( tile->failov.vote_account_path, config->paths.vote_account, sizeof(tile->failov.vote_account_path) ); - if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->firedancer.failover.listen_address, &tile->failov.listen_addr ) ) ) + fd_cstr_ncpy( tile->failov.identity_key_path, identity_key_path, sizeof(tile->failov.identity_key_path) ); + fd_cstr_ncpy( tile->failov.staked_key_path, config->firedancer.failover.enabled ? config->paths.identity_key : "", sizeof(tile->failov.staked_key_path) ); + fd_cstr_ncpy( tile->failov.vote_account_path, config->paths.vote_account, sizeof(tile->failov.vote_account_path) ); + if( FD_UNLIKELY( config->firedancer.failover.enabled && !fd_cstr_to_ip4_addr( config->firedancer.failover.listen_address, &tile->failov.listen_addr ) ) ) FD_LOG_ERR(( "[failover.listen_address] `%s` must be an IPv4 address, host names are not accepted", config->firedancer.failover.listen_address )); tile->failov.port = config->firedancer.failover.listen_port; fd_cstr_ncpy( tile->failov.gossip_host, config->firedancer.gossip.host, sizeof(tile->failov.gossip_host) ); @@ -1852,8 +1852,8 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) ) ) { fd_cstr_ncpy( tile->sign.identity_key_path, identity_key_path, sizeof(tile->sign.identity_key_path) ); - tile->sign.failover_enabled = config->firedancer.failover.enabled; - fd_cstr_ncpy( tile->sign.failover_staked_key_path, config->paths.identity_key, sizeof(tile->sign.failover_staked_key_path) ); + tile->sign.failover_enabled = fd_topo_find_tile( &config->topo, "failov", 0UL )!=ULONG_MAX; + fd_cstr_ncpy( tile->sign.failover_staked_key_path, config->firedancer.failover.enabled ? config->paths.identity_key : "", sizeof(tile->sign.failover_staked_key_path) ); tile->sign.authorized_voter_paths_cnt = config->firedancer.paths.authorized_voter_paths_cnt; for( ulong i=0UL; isign.authorized_voter_paths_cnt; i++ ) { diff --git a/src/app/shared/commands/run/run.c b/src/app/shared/commands/run/run.c index 139c4d02739..73b61f52a4c 100644 --- a/src/app/shared/commands/run/run.c +++ b/src/app/shared/commands/run/run.c @@ -82,7 +82,7 @@ run_cmd_perm( args_t * args, fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for serving metrics" ); if( FD_UNLIKELY( config->tiles.gui.gui_listen_port<1024 ) ) fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for serving the GUI" ); - if( FD_UNLIKELY( config->is_firedancer && config->firedancer.failover.enabled && config->firedancer.failover.listen_port<1024 ) ) + if( FD_UNLIKELY( config->firedancer.failover.enabled && config->firedancer.failover.listen_port<1024 ) ) fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for the failover listener" ); } @@ -1076,7 +1076,10 @@ failover_junk_write( char const * tmp ) { uchar keypair[ 64 ]; for( long got=0L; got<32L; ) { long n = getrandom( keypair+got, (ulong)(32L-got), GRND_RANDOM ); - if( FD_UNLIKELY( -1==n ) ) FD_LOG_ERR(( "could not create the failover junk identity, getrandom() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( -1==n ) ) { + if( FD_LIKELY( errno==EINTR ) ) continue; + FD_LOG_ERR(( "could not create the failover junk identity, getrandom() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } got += n; } fd_sha512_t sha[ 1 ]; @@ -1095,9 +1098,17 @@ failover_junk_write( char const * tmp ) { int fd = open( tmp, O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC, S_IRUSR|S_IWUSR ); if( FD_UNLIKELY( -1==fd ) ) FD_LOG_ERR(( "could not create `%s` (%i-%s)", tmp, errno, fd_io_strerror( errno ) )); - long written = write( fd, json, len ); + for( ulong written=0UL; writtenfailover.enabled ) ) { - jw_obj_open( &w, "failover" ); - jw_bool ( &w, "enabled", f->failover.enabled ); - jw_path ( &w, "listen_address", f->failover.listen_address ); - jw_ulong( &w, "listen_port", f->failover.listen_port ); - jw_obj_close( &w ); - } + jw_obj_open( &w, "failover" ); + jw_bool ( &w, "enabled", f->failover.enabled ); + jw_path ( &w, "listen_address", f->failover.listen_address ); + jw_ulong( &w, "listen_port", f->failover.listen_port ); + jw_obj_close( &w ); jw_obj_open( &w, "hugetlbfs" ); jw_path ( &w, "gigantic_page_mount_path", config->hugetlbfs.gigantic_page_mount_path ); diff --git a/src/app/shared/test_config_json.c b/src/app/shared/test_config_json.c index 92db78ef362..a34f3b3d8d0 100644 --- a/src/app/shared/test_config_json.c +++ b/src/app/shared/test_config_json.c @@ -139,8 +139,8 @@ main( int argc, FD_TEST( !strstr( json, "hunter2" ) ); FD_TEST( !strstr( json, "example.com" ) ); - /* without failover nothing about it is reported */ - FD_TEST( !strstr( json, "\"failover\"" ) ); + /* failover settings are reported even when disabled */ + FD_TEST( strstr( json, "\"failover\":{\"enabled\":false,\"listen_address\":\"[redacted]\",\"listen_port\":8010}" ) ); /* the listen address is redacted, the rest is reported */ config->firedancer.failover.enabled = 1; diff --git a/src/app/shared/test_config_parse.c b/src/app/shared/test_config_parse.c index 0e8ef7222ad..d55ba72ddee 100644 --- a/src/app/shared/test_config_parse.c +++ b/src/app/shared/test_config_parse.c @@ -76,6 +76,7 @@ vote_history_path_is_valid( char const * path ) { return WIFEXITED( status ) && !WEXITSTATUS( status ); } + /* Runs fd_config_validate_consensus on the given combination and reports whether it was accepted. It exits the process on rejection, so run it in a child. */ diff --git a/src/disco/events/generated/fd_event_gen.h b/src/disco/events/generated/fd_event_gen.h index 9053ce9e663..798780a21c5 100644 --- a/src/disco/events/generated/fd_event_gen.h +++ b/src/disco/events/generated/fd_event_gen.h @@ -517,7 +517,8 @@ typedef struct fd_event_snapshot_created fd_event_snapshot_created_t; #define FD_EVENT_ADMIN_COMMAND_TYPE_ADD_AUTHORIZED_VOTER (3) /* Add an authorized voter key. */ #define FD_EVENT_ADMIN_COMMAND_TYPE_REMOVE_ALL_AUTHORIZED_VOTERS (4) /* Remove every authorized voter key. */ #define FD_EVENT_ADMIN_COMMAND_TYPE_SNAPSHOT_CREATE (5) /* Request creation of a snapshot. */ -#define FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER (6) /* Run a failover command. */ +#define FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE (6) /* Promote the running standby validator to active. */ +#define FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_STATUS (7) /* Read the running validator's failover state. */ /* The result of processing the admin command. */ #define FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS (1) /* The command completed successfully. */ diff --git a/src/disco/events/schema/admin_command.json b/src/disco/events/schema/admin_command.json index 01fb9e8ef12..f6ffe9e672d 100644 --- a/src/disco/events/schema/admin_command.json +++ b/src/disco/events/schema/admin_command.json @@ -12,7 +12,8 @@ "add_authorized_voter": { "description": "Add an authorized voter key." }, "remove_all_authorized_voters": { "description": "Remove every authorized voter key." }, "snapshot_create": { "description": "Request creation of a snapshot." }, - "failover": { "description": "Run a failover command." } + "failover_promote": { "description": "Promote the running standby validator to active." }, + "failover_status": { "description": "Read the running validator's failover state." } } }, "result": { diff --git a/src/disco/events/schema/events.proto b/src/disco/events/schema/events.proto index fcad7278760..d1c204b265c 100644 --- a/src/disco/events/schema/events.proto +++ b/src/disco/events/schema/events.proto @@ -405,7 +405,8 @@ enum AdminCommandType { ADMIN_COMMAND_TYPE_ADD_AUTHORIZED_VOTER = 3; // Add an authorized voter key. ADMIN_COMMAND_TYPE_REMOVE_ALL_AUTHORIZED_VOTERS = 4; // Remove every authorized voter key. ADMIN_COMMAND_TYPE_SNAPSHOT_CREATE = 5; // Request creation of a snapshot. - ADMIN_COMMAND_TYPE_FAILOVER = 6; // Run a failover command. + ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE = 6; // Promote the running standby validator to active. + ADMIN_COMMAND_TYPE_FAILOVER_STATUS = 7; // Read the running validator's failover state. } // The result of processing the admin command. diff --git a/src/disco/keyguard/fd_keyguard.h b/src/disco/keyguard/fd_keyguard.h index 161afd2c3f2..2b20cbd9c8e 100644 --- a/src/disco/keyguard/fd_keyguard.h +++ b/src/disco/keyguard/fd_keyguard.h @@ -40,7 +40,8 @@ FD_STATIC_ASSERT( FD_KEYGUARD_AUTH_VOTERS_MAX<=16UL, auth_voters_max_fits_sign_r #define FD_KEYGUARD_ROLE_CNT (12) /* number of known roles */ /* The failov tile gets its member certificate signed, a plain Ed25519 - signature by the staked key over a 16 byte domain prefix followed by + signature by the failover identity over a 16 byte domain prefix + followed by the sign tile's own junk pubkey. The prefix keeps that signature out of every other raw message domain, in particular the 32 byte shred roots the leader role signs. The junk key signs its TLS diff --git a/src/disco/keyguard/fd_keyswitch.h b/src/disco/keyguard/fd_keyswitch.h index 4a9cc3c04bf..b546a1cb3c5 100644 --- a/src/disco/keyguard/fd_keyswitch.h +++ b/src/disco/keyguard/fd_keyswitch.h @@ -21,10 +21,14 @@ /* Application-specific param values should be defined below. */ /* Sign tile identity switches. KEYPAIR passes a new keypair in - bytes[0..64), PUBKEY selects a failover key loaded at boot by its - public key in bytes[0..32). */ + bytes[0..64), PUBKEY passes a public key in bytes[0..32) and selects + the junk key or the failover identity the sign tile already holds. + CHECK passes a public key in bytes[0..32) and checks whether it can + become the failover identity without changing any keys. */ #define FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR (0UL) #define FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY (1UL) +#define FD_KEYSWITCH_PARAM_IDENTITY_CHECK (2UL) +#define FD_KEYSWITCH_PARAM_IDENTITY_PREPARE (3UL) /* preload a failover key without installing it */ #define FD_KEYSWITCH_PARAM_AV_ADD (0UL) #define FD_KEYSWITCH_PARAM_AV_CLEAR (1UL) @@ -35,6 +39,7 @@ struct __attribute__((aligned(FD_KEYSWITCH_ALIGN))) fd_keyswitch_private { ulong result; ulong param; uchar bytes[ 32768UL ]; + ulong operator; /* 1 when set-identity made this switch under failover, so the tower or votor votes without waiting for an adoption */ /* Padding to FD_KEYSWITCH_ALIGN here */ }; diff --git a/src/disco/keyguard/fd_sign_tile.c b/src/disco/keyguard/fd_sign_tile.c index 2a91eb9d21d..239322ac046 100644 --- a/src/disco/keyguard/fd_sign_tile.c +++ b/src/disco/keyguard/fd_sign_tile.c @@ -63,8 +63,10 @@ typedef struct { uchar * public_key; uchar * private_key; - uchar const * failover_junk_key; /* read only copies loaded at boot under failover */ + uchar const * failover_junk_key; /* read only copies loaded at boot under failover */ uchar const * failover_staked_key; + uchar * failover_operator_key; /* a set-identity key other than junk and staked, zero while there is none */ + uchar const * failover_key; /* the failover identity, failover_staked_key or failover_operator_key */ fd_keyguard_bls_key_t * bls_keys; @@ -105,17 +107,17 @@ derive_fields( fd_sign_ctx_t * ctx ) { fd_keyguard_bls_key_derive( &ctx->bls_keys[ 0 ], ctx->public_key, ctx->private_key, ctx->sha512 ); } -/* A failover switch passes only a public key, we copy the matching - key loaded at boot into the identity key. */ +/* A failover switch passes only a public key, we copy the junk key or + the failover identity into the identity key. */ static void FD_FN_SENSITIVE failover_select( fd_sign_ctx_t * ctx ) { uchar const * selected = NULL; - if( FD_LIKELY( ctx->failover_staked_key ) ) { - if( fd_memeq( ctx->keyswitch->bytes, ctx->failover_junk_key+32UL, 32UL ) ) selected = ctx->failover_junk_key; - else if( fd_memeq( ctx->keyswitch->bytes, ctx->failover_staked_key+32UL, 32UL ) ) selected = ctx->failover_staked_key; + if( FD_LIKELY( ctx->failover_key ) ) { + if( fd_memeq( ctx->keyswitch->bytes, ctx->failover_junk_key+32UL, 32UL ) ) selected = ctx->failover_junk_key; + else if( fd_memeq( ctx->keyswitch->bytes, ctx->failover_key+32UL, 32UL ) ) selected = ctx->failover_key; } - if( FD_UNLIKELY( !selected ) ) FD_LOG_ERR(( "identity switch selects a key that was not loaded at boot" )); + if( FD_UNLIKELY( !selected ) ) FD_LOG_ERR(( "identity switch selects a key that is neither the junk key nor the failover identity" )); fd_memzero_explicit( ctx->keyswitch->bytes, 64UL ); fd_memcpy( ctx->private_key, selected, 32UL ); fd_memcpy( ctx->public_key, selected+32UL, 32UL ); @@ -123,31 +125,77 @@ failover_select( fd_sign_ctx_t * ctx ) { fd_keyswitch_state( ctx->keyswitch, FD_KEYSWITCH_STATE_COMPLETED ); } -/* failover_housekeeping runs the keyswitches the failover way before - the normal keyswitch code sees them. A failover switch selects a key - loaded at boot. */ +static int +failover_identity_allowed( fd_sign_ctx_t const * ctx, + uchar const * pubkey ) { + /* Selecting junk keeps the existing failover identity. */ + if( fd_memeq( pubkey, ctx->failover_junk_key+32UL, 32UL ) ) return 1; + for( ulong i=0UL; iauthorized_voters_cnt; i++ ) { + if( fd_memeq( pubkey, ctx->authorized_voter_pubkeys[ i ], 32UL ) ) return 0; + } + return 1; +} -static void FD_FN_SENSITIVE +/* failover_housekeeping runs the keyswitches the failover way before + the normal keyswitch code sees them. A failover switch selects the junk key + or the failover identity. set-identity also makes the new key the one + failover hands off, unless it is our junk key. The staked key keeps its + read only page from boot, another key is copied into its own, and the + normal switch installs it. Return 0 only when the normal switch must + install this request. Once acknowledged, the keyswitch belongs to + Admin and must not be read again in this housekeeping call. */ + +static int FD_FN_SENSITIVE failover_housekeeping( fd_sign_ctx_t * ctx ) { - if( FD_UNLIKELY( fd_keyswitch_state_query( ctx->keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING && - fd_keyswitch_param_query( ctx->keyswitch )==FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY ) ) { + if( FD_LIKELY( fd_keyswitch_state_query( ctx->keyswitch )!=FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) return 1; + ulong param = fd_keyswitch_param_query( ctx->keyswitch ); + if( FD_UNLIKELY( param==FD_KEYSWITCH_PARAM_IDENTITY_PREPARE ) ) { + int allowed = failover_identity_allowed( ctx, ctx->keyswitch->bytes+32UL ) && + !fd_memeq( ctx->keyswitch->bytes+32UL, ctx->failover_junk_key+32UL, 32UL ); + if( FD_LIKELY( allowed ) ) { + fd_memcpy( ctx->failover_operator_key, ctx->keyswitch->bytes, 64UL ); + ctx->failover_key = ctx->failover_operator_key; + } + fd_memzero_explicit( ctx->keyswitch->bytes, 64UL ); + ctx->keyswitch->result = allowed ? FD_ADMINCTL_RESULT_SUCCESS : FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER; + fd_keyswitch_state( ctx->keyswitch, allowed ? FD_KEYSWITCH_STATE_COMPLETED : FD_KEYSWITCH_STATE_FAILED ); + return 1; + } + /* Before a client prepares a target, set-identity retains its normal + behavior and does not select an identity for failover. */ + if( param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR && !ctx->failover_key ) return 0; + if( FD_UNLIKELY( param==FD_KEYSWITCH_PARAM_IDENTITY_CHECK || param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR ) ) { + uchar const * pubkey = ctx->keyswitch->bytes + (param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR ? 32UL : 0UL); + int allowed = failover_identity_allowed( ctx, pubkey ); + if( FD_UNLIKELY( !allowed || param==FD_KEYSWITCH_PARAM_IDENTITY_CHECK ) ) { + fd_memzero_explicit( ctx->keyswitch->bytes, 64UL ); + ctx->keyswitch->result = allowed ? FD_ADMINCTL_RESULT_SUCCESS : FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER; + fd_keyswitch_state( ctx->keyswitch, allowed ? FD_KEYSWITCH_STATE_COMPLETED : FD_KEYSWITCH_STATE_FAILED ); + return 1; + } + } + if( FD_UNLIKELY( param==FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY ) ) { failover_select( ctx ); + return 1; } - if( FD_UNLIKELY( ctx->av_keyswitch && fd_keyswitch_state_query( ctx->av_keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING && - fd_keyswitch_param_query( ctx->av_keyswitch )==FD_KEYSWITCH_PARAM_AV_ADD && - fd_memeq( ctx->av_keyswitch->bytes+32UL, ctx->failover_staked_key+32UL, 32UL ) ) ) { - FD_LOG_WARNING(( "keyswitch failed: the staked [paths.identity_key] key cannot be an authorized voter under failover" )); - fd_memzero_explicit( ctx->av_keyswitch->bytes, 64UL ); - ctx->av_keyswitch->result = FD_ADMINCTL_RESULT_UNSUPPORTED; - fd_keyswitch_state( ctx->av_keyswitch, FD_KEYSWITCH_STATE_FAILED ); + FD_TEST( param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR ); + if( FD_UNLIKELY( !fd_memeq( ctx->keyswitch->bytes+32UL, ctx->failover_junk_key+32UL, 32UL ) ) ) { + if( ctx->failover_staked_key && fd_memeq( ctx->keyswitch->bytes+32UL, ctx->failover_staked_key+32UL, 32UL ) ) { + ctx->failover_key = ctx->failover_staked_key; + fd_memzero_explicit( ctx->failover_operator_key, 64UL ); + } else { + fd_memcpy( ctx->failover_operator_key, ctx->keyswitch->bytes, 64UL ); + ctx->failover_key = ctx->failover_operator_key; + } } + return 0; } static void FD_FN_SENSITIVE during_housekeeping_sensitive( fd_sign_ctx_t * ctx ) { - if( FD_UNLIKELY( ctx->failover_staked_key ) ) failover_housekeeping( ctx ); + int identity_handled = ctx->failover_junk_key && failover_housekeeping( ctx ); - if( FD_UNLIKELY( fd_keyswitch_state_query( ctx->keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) { + if( FD_UNLIKELY( !identity_handled && fd_keyswitch_state_query( ctx->keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) { memcpy( ctx->private_key, ctx->keyswitch->bytes, 32UL ); fd_memzero_explicit( ctx->keyswitch->bytes, 32UL ); FD_COMPILER_MFENCE(); @@ -162,6 +210,13 @@ during_housekeeping_sensitive( fd_sign_ctx_t * ctx ) { if( FD_UNLIKELY( ctx->av_keyswitch && fd_keyswitch_state_query( ctx->av_keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) { ulong param = fd_keyswitch_param_query( ctx->av_keyswitch ); if( FD_LIKELY( param==FD_KEYSWITCH_PARAM_AV_ADD ) ) { + if( FD_UNLIKELY( ctx->failover_key && fd_memeq( ctx->av_keyswitch->bytes+32UL, ctx->failover_key+32UL, 32UL ) ) ) { + FD_LOG_WARNING(( "keyswitch failed: the failover identity cannot be an authorized voter" )); + fd_memzero_explicit( ctx->av_keyswitch->bytes, 64UL ); + ctx->av_keyswitch->result = FD_ADMINCTL_RESULT_UNSUPPORTED; + fd_keyswitch_state( ctx->av_keyswitch, FD_KEYSWITCH_STATE_FAILED ); + return; + } if( FD_UNLIKELY( ctx->authorized_voters_cnt==FD_KEYGUARD_AUTH_VOTERS_MAX ) ) { FD_LOG_WARNING(( "keyswitch failed: maximum number of authorized voters reached" )); fd_memzero_explicit( ctx->av_keyswitch->bytes, 64UL ); @@ -252,9 +307,9 @@ during_frag( void * _ctx, } /* The failover tile's ED25519 requests. The junk key signs the - failover TLS whichever key is installed. The staked key signs the - member certificate whichever key is installed, but only over our own - junk pubkey. */ + failover TLS whichever key is installed. The failover identity signs + the member certificate whichever key is installed, but only over our + own junk pubkey. */ static void FD_FN_SENSITIVE failover_sign( fd_sign_ctx_t * ctx, @@ -267,7 +322,8 @@ failover_sign( fd_sign_ctx_t * ctx, if( FD_UNLIKELY( !fd_memeq( ctx->_data+FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ, ctx->failover_junk_key+32UL, 32UL ) ) ) { FD_LOG_EMERG(( "member certificate request is not for our junk key" )); } - fd_ed25519_sign( dst, ctx->_data, sz, ctx->failover_staked_key+32UL, ctx->failover_staked_key, ctx->sha512 ); + if( FD_UNLIKELY( !ctx->failover_key ) ) FD_LOG_EMERG(( "member certificate requested before preparing a failover identity" )); + fd_ed25519_sign( dst, ctx->_data, sz, ctx->failover_key+32UL, ctx->failover_key, ctx->sha512 ); } static void FD_FN_SENSITIVE @@ -402,20 +458,26 @@ after_frag( void * _ctx, /* failover_load_keys sets the failover keys, all NULL unless failover is enabled. Under failover we also keep read only copies of the junk - and staked keypairs, loaded here before the sandbox. We boot with the - junk key as the identity key. */ + and staked keypairs, loaded here before the sandbox, and a page for a + key that set-identity installs. We boot with the junk key as the + identity key. */ static void FD_FN_SENSITIVE failover_load_keys( fd_sign_ctx_t * ctx, fd_topo_tile_t const * tile ) { - ctx->failover_junk_key = NULL; - ctx->failover_staked_key = NULL; + ctx->failover_junk_key = NULL; + ctx->failover_staked_key = NULL; + ctx->failover_operator_key = NULL; + ctx->failover_key = NULL; if( FD_LIKELY( !tile->sign.failover_enabled ) ) return; uchar * junk_key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); fd_memcpy( junk_key, ctx->private_key, 64UL ); - ctx->failover_junk_key = fd_keyload_mprotect_ro( junk_key, /* public_key_only: */ 0 ); - ctx->failover_staked_key = fd_keyload_load( tile->sign.failover_staked_key_path, /* pubkey only: */ 0 ); + ctx->failover_junk_key = fd_keyload_mprotect_ro( junk_key, /* public_key_only: */ 0 ); + ctx->failover_operator_key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); + if( !tile->sign.failover_staked_key_path[ 0 ] ) return; + ctx->failover_staked_key = fd_keyload_load( tile->sign.failover_staked_key_path, /* pubkey only: */ 0 ); + ctx->failover_key = ctx->failover_staked_key; if( FD_UNLIKELY( fd_memeq( ctx->failover_junk_key+32UL, ctx->failover_staked_key+32UL, 32UL ) ) ) { FD_LOG_ERR(( "the failover junk identity must differ from the [paths.identity_key] key, which is the staked identity under failover" )); } @@ -581,7 +643,7 @@ unprivileged_init_sensitive( fd_topo_t const * topo, FD_TEST( !strcmp( out_link->name, "sign_failov" ) ); FD_TEST( in_link->mtu==130UL ); FD_TEST( out_link->mtu==64UL ); - FD_TEST( ctx->failover_staked_key ); + FD_TEST( ctx->failover_junk_key ); } else { FD_LOG_CRIT(( "unexpected link %s", in_link->name )); } diff --git a/src/disco/keyguard/test_sign_tile.c b/src/disco/keyguard/test_sign_tile.c index 8eace67821d..952533092cf 100644 --- a/src/disco/keyguard/test_sign_tile.c +++ b/src/disco/keyguard/test_sign_tile.c @@ -296,11 +296,13 @@ test_bls_request_rejected( void ) { FD_TEST( bls_request_exit_status( FD_KEYGUARD_SIGN_TYPE_BLS|(1UL<<32), 11UL )==1 ); /* authorized voter 0, none loaded */ } -/* Failover: the sign tile holds the junk and staked keypairs it loaded - at boot and switches between them by public key. */ +/* Under failover the sign tile switches by public key between the junk + key and the failover identity, the staked key from boot or a key + set-identity installed. */ -static uchar junk_key [ 64 ]; -static uchar staked_key[ 64 ]; +static uchar junk_key [ 64 ]; +static uchar staked_key [ 64 ]; +static uchar operator_key[ 64 ]; /* the sign tile's page for a set-identity key */ static void failover_setup( void ) { @@ -311,8 +313,11 @@ failover_setup( void ) { fd_ed25519_public_from_private( staked_key+32UL, staked_key, ctx.sha512 ); fd_memcpy( identity_key, junk_key, 64UL ); /* a failover member boots under the junk key */ derive_fields( &ctx ); - ctx.failover_junk_key = junk_key; - ctx.failover_staked_key = staked_key; + fd_memset( operator_key, 0, 64UL ); + ctx.failover_junk_key = junk_key; + ctx.failover_staked_key = staked_key; + ctx.failover_operator_key = operator_key; + ctx.failover_key = staked_key; ctx.in[0].role = FD_KEYGUARD_ROLE_FAILOV; client.response_mtu = FD_ED25519_SIG_SZ; } @@ -412,7 +417,7 @@ test_failover_keys( void ) { if( i ) failover_switch( identity_switch, installed+32UL ); FD_TEST( fd_memeq( ctx.public_key, installed+32UL, 32UL ) ); FD_TEST( fd_memeq( ctx.private_key, installed, 32UL ) ); - FD_TEST( fd_memeq( keys[0].public_key, bls->public_key, FD_KEYGUARD_BLS_PUBKEY_SZ ) ); + FD_TEST( fd_memeq( keys[0].public_key, bls->public_key, FD_BLS_PUB_COMPRESSED_SZ ) ); FD_TEST( failover_signed_by( cert, sizeof(cert), staked_key+32UL ) ); FD_TEST( failover_signed_by( cv, sizeof(cv), junk_key +32UL ) ); } @@ -431,6 +436,83 @@ test_failover_keys( void ) { FD_LOG_NOTICE(( "pass: failover keys" )); } +/* A set-identity keypair other than the junk key becomes the failover + identity, it signs the member certificate and cannot become an + authorized voter. The staked key keeps its boot page, another key goes in the + operator page. */ + +static void +operator_switch( fd_keyswitch_t * ks, + uchar const * keypair ) { + fd_memcpy( ks->bytes, keypair, 64UL ); + ks->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR; + fd_keyswitch_state( ks, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( ks )==FD_KEYSWITCH_STATE_COMPLETED ); +} + +static void +test_failover_operator_key( void ) { + failover_setup(); + fd_keyswitch_t identity_switch[1]; + ctx.keyswitch = fd_keyswitch_join( fd_keyswitch_new( identity_switch, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.keyswitch ); + uchar third[ 64 ]; + fd_memset( third, 0x66, 32UL ); + fd_ed25519_public_from_private( third+32UL, third, ctx.sha512 ); + uchar cert[ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ]; + fd_memcpy( cert, FD_KEYGUARD_MEMBER_CERT_PREFIX, FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ ); + fd_memcpy( cert+FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ, junk_key+32UL, 32UL ); + + uchar staked_copy[ 64 ]; + fd_memcpy( staked_copy, staked_key, 64UL ); + operator_switch( identity_switch, third ); + FD_TEST( fd_memeq( ctx.public_key, third+32UL, 32UL ) && fd_memeq( operator_key, third, 64UL ) ); + FD_TEST( ctx.failover_key==operator_key && fd_memeq( staked_key, staked_copy, 64UL ) ); + FD_TEST( failover_signed_by( cert, sizeof(cert), third+32UL ) ); + + /* A failover switch can now select the new key and the junk key. */ + failover_switch( identity_switch, junk_key+32UL ); + FD_TEST( fd_memeq( ctx.public_key, junk_key+32UL, 32UL ) ); + failover_switch( identity_switch, third+32UL ); + FD_TEST( fd_memeq( ctx.public_key, third+32UL, 32UL ) ); + + /* set-identity to the junk key keeps the failover identity. */ + operator_switch( identity_switch, junk_key ); + FD_TEST( fd_memeq( ctx.public_key, junk_key+32UL, 32UL ) && ctx.failover_key==operator_key && fd_memeq( operator_key, third, 64UL ) ); + FD_TEST( failover_signed_by( cert, sizeof(cert), third+32UL ) ); + + /* The failover identity cannot become an authorized voter, the + configured key may be one now. */ + fd_keyswitch_t voter_switch[1]; + ctx.av_keyswitch = fd_keyswitch_join( fd_keyswitch_new( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); + FD_TEST( ctx.av_keyswitch ); + fd_memcpy( voter_switch->bytes, third, 64UL ); + voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_FAILED && !ctx.authorized_voters_cnt ); + fd_keyswitch_state( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + fd_memcpy( voter_switch->bytes, staked_key, 64UL ); + voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; + during_housekeeping( &ctx ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_COMPLETED && ctx.authorized_voters_cnt==1UL ); + + /* After removing the voter, set-identity can select the staked key. + It goes back to its boot page and clears the operator page. */ + voter_switch->param = FD_KEYSWITCH_PARAM_AV_CLEAR; + fd_keyswitch_state( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( !ctx.authorized_voters_cnt ); + operator_switch( identity_switch, staked_key ); + uchar zero[ 64 ] = {0}; + FD_TEST( ctx.failover_key==staked_key && fd_memeq( operator_key, zero, 64UL ) ); + FD_TEST( failover_signed_by( cert, sizeof(cert), staked_key+32UL ) ); + failover_switch( identity_switch, junk_key+32UL ); + failover_switch( identity_switch, staked_key+32UL ); + FD_TEST( fd_memeq( ctx.public_key, staked_key+32UL, 32UL ) ); + FD_LOG_NOTICE(( "pass: failover operator key" )); +} + static fd_keyswitch_t failover_ks[1]; static void @@ -463,8 +545,9 @@ child_exit_status( void (*fn)( void ) ) { return WEXITSTATUS( status ); } -/* A switch to a key not loaded at boot, and a certificate over another - junk key, stop the validator. */ +/* A switch to a key that is neither the junk key nor the failover + identity, and a certificate over another junk key, stop the + validator. */ static void test_failover_refusals( void ) { @@ -492,6 +575,7 @@ main( int argc, test_bls_request_rejected(); test_identity_switch_without_failover(); test_failover_keys(); + test_failover_operator_key(); test_failover_refusals(); FD_LOG_NOTICE(( "pass" )); fd_halt(); diff --git a/src/disco/topo/fd_topo.c b/src/disco/topo/fd_topo.c index 55132fb5865..1076d03be0e 100644 --- a/src/disco/topo/fd_topo.c +++ b/src/disco/topo/fd_topo.c @@ -223,9 +223,9 @@ FD_FN_PURE static ulong fd_topo_tile_extra_normal_pages( fd_topo_tile_t const * tile ) { ulong key_pages = 0UL; if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) && tile->sign.failover_enabled ) ) { - /* The read only junk and staked key pages and their guards, next - to the identity key. */ - key_pages += 10UL; + /* The read only junk and staked key pages, the set-identity key + page, and their guards, next to the identity key. */ + key_pages += 15UL; } if( FD_UNLIKELY( tile->id_keyswitch_obj_id!=ULONG_MAX ) ) { /* Certain tiles using fd_keyload_load need normal pages to hold diff --git a/src/discof/admin/fd_admin_tile.c b/src/discof/admin/fd_admin_tile.c index 77d09032852..8eb5bab1e48 100644 --- a/src/discof/admin/fd_admin_tile.c +++ b/src/discof/admin/fd_admin_tile.c @@ -25,7 +25,13 @@ struct fd_admin_tile_ctx { long poll_ticks; long idle_poll_ticks; long deadline_ticks; /* next adminctl poll */ - int failover_enabled; /* failover moves the identity, set-identity is refused */ + + int failover_enabled; /* failover hands off the identity */ + int failover_switch; /* a failover identity switch is in progress */ + ulong operator_epoch; /* failover epoch, bumped by every set-identity under failover */ + int operator_pending; /* the failover tile has not been told about the last set-identity yet */ + uchar junk_pubkey[ 32 ]; /* the identity we booted with, the junk key under failover */ + uchar failover_pubkey[ 32 ]; /* the key failover hands off, [paths.identity_key] until set-identity installs another */ /* Failover commands go to the failover tile over the bus, one at a time like snapshot creation. */ @@ -113,6 +119,15 @@ privileged_init( fd_topo_t const * topo, FD_LOG_ERR(( "identity_key_path not set" )); fd_memcpy( ctx->identity_pubkey, fd_keyload_load( tile->admin.identity_key_path, /* pubkey only: */ 1 ), 32UL ); + + /* Under failover we boot on the junk key, and the key failover hands off + starts as [paths.identity_key]. */ + ulong failov_idx = fd_topo_find_tile( topo, "failov", 0UL ); + if( FD_UNLIKELY( failov_idx!=ULONG_MAX ) ) { + fd_memcpy( ctx->junk_pubkey, ctx->identity_pubkey, 32UL ); + char const * path = topo->tiles[ failov_idx ].failov.staked_key_path; + if( path[ 0 ] ) fd_memcpy( ctx->failover_pubkey, fd_keyload_load( path, /* pubkey only: */ 1 ), 32UL ); + } } static void @@ -125,9 +140,10 @@ unprivileged_init( fd_topo_t const * topo, ctx->poll_ticks = (long)(1e6*fd_tempo_tick_per_ns( NULL )); /* 1ms */ ctx->idle_poll_ticks = (long)((double)FD_SLEEP_PARK_CAP_NS*fd_tempo_tick_per_ns( NULL )); ctx->deadline_ticks = 0L; - ctx->failover_enabled = fd_topo_find_tile( topo, "failov", 0UL )!=ULONG_MAX; + ulong failov_idx = fd_topo_find_tile( topo, "failov", 0UL ); + ctx->failover_enabled = failov_idx!=ULONG_MAX && topo->tiles[ failov_idx ].failov.staked_key_path[ 0 ]; ctx->failover_slot_idx = ULONG_MAX; - ctx->topo = topo; + ctx->topo = topo; fd_topo_obj_t const * adminctl_obj = fd_topo_find_tile_obj( topo, tile, "adminctl" ); FD_TEST( adminctl_obj ); @@ -334,8 +350,8 @@ unprivileged_init( fd_topo_t const * topo, static fd_keyswitch_t * find_identity_keyswitch( fd_admin_tile_ctx_t * ctx, char const * tile_name ) { - fd_topo_t const * topo = ctx->topo; - ulong tile_idx = fd_topo_find_tile( topo, tile_name, 0UL ); + fd_topo_t const * topo = ctx->topo; + ulong tile_idx = fd_topo_find_tile( topo, tile_name, 0UL ); FD_TEST( tile_idx!=ULONG_MAX ); FD_TEST( topo->tiles[ tile_idx ].id_keyswitch_obj_id!=ULONG_MAX ); @@ -473,9 +489,9 @@ poll_set_identity( fd_admin_tile_ctx_t * ctx, if( strcmp( tile->name, "sign" ) ) continue; fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id ); memcpy( sign->bytes, keypair, 64UL ); - if( FD_UNLIKELY( ctx->failover_enabled ) ) { - /* Under failover the sign tile selects a keypair it loaded at - boot by its public key. */ + if( FD_UNLIKELY( ctx->failover_switch ) ) { + /* Under failover the sign tile selects the junk key or the + failover identity by its public key. */ sign->param = FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY; fd_memcpy( sign->bytes, keypair+32UL, 32UL ); fd_memset( sign->bytes+32UL, 0, 32UL ); @@ -625,8 +641,91 @@ poll_set_identity( fd_admin_tile_ctx_t * ctx, return *state==FD_SET_IDENTITY_STATE_UNLOCKED; } +/* Check every sign tile before changing the failover identity or halting + any tiles. Admin serializes commands, so authorized voters cannot + change between this check and the identity switch. */ + +static void +failover_identity_check_begin( fd_admin_tile_ctx_t * ctx, + uchar const * identity ) { + fd_topo_t const * topo = ctx->topo; + for( ulong i=0UL; itile_cnt; i++ ) { + fd_topo_tile_t const * tile = &topo->tiles[ i ]; + if( strcmp( tile->name, "sign" ) ) continue; + fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id ); + fd_memcpy( sign->bytes, identity, 32UL ); + sign->param = FD_KEYSWITCH_PARAM_IDENTITY_CHECK; + fd_keyswitch_state( sign, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + } +} + +/* Returns ULONG_MAX until every sign tile has answered. */ +static ulong +failover_identity_check_poll( fd_admin_tile_ctx_t * ctx ) { + fd_topo_t const * topo = ctx->topo; + ulong result = FD_ADMINCTL_RESULT_SUCCESS; + for( ulong i=0UL; itile_cnt; i++ ) { + fd_topo_tile_t const * tile = &topo->tiles[ i ]; + if( strcmp( tile->name, "sign" ) ) continue; + fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id ); + ulong state = fd_keyswitch_state_query( sign ); + if( state==FD_KEYSWITCH_STATE_SWITCH_PENDING ) return ULONG_MAX; + FD_TEST( state==FD_KEYSWITCH_STATE_COMPLETED || state==FD_KEYSWITCH_STATE_FAILED ); + if( state==FD_KEYSWITCH_STATE_FAILED ) result = sign->result; + } + for( ulong i=0UL; itile_cnt; i++ ) { + fd_topo_tile_t const * tile = &topo->tiles[ i ]; + if( strcmp( tile->name, "sign" ) ) continue; + fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id ); + sign->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR; + fd_keyswitch_state( sign, FD_KEYSWITCH_STATE_UNLOCKED ); + } + return result; +} + +/* Under failover set-identity still switches the same way. */ + +static void +failover_operator_begin( fd_admin_tile_ctx_t * ctx, + uchar const * identity ) { + ctx->operator_epoch++; + if( !fd_memeq( identity, ctx->junk_pubkey, 32UL ) ) fd_memcpy( ctx->failover_pubkey, identity, 32UL ); + FD_BASE58_ENCODE_32_BYTES( identity, new_identity ); + FD_BASE58_ENCODE_32_BYTES( ctx->failover_pubkey, failover_identity ); + FD_LOG_WARNING(( "set-identity switches to `%s` while failover is on, failover restarts on this machine for identity `%s`", new_identity, failover_identity )); + + fd_topo_t const * topo = ctx->topo; + for( ulong i=0UL; itile_cnt; i++ ) { + if( strcmp( topo->tiles[ i ].name, "sign" ) ) continue; + fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, topo->tiles[ i ].id_keyswitch_obj_id ); + sign->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR; + } + find_identity_keyswitch( ctx, "tower" )->operator = 1UL; + ctx->operator_pending = 1; +} + +/* Sends a pending OPERATOR once the failover link has room to spare. */ + +static void +failover_operator_notify( fd_admin_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + if( FD_LIKELY( !ctx->operator_pending ) ) return; + if( FD_UNLIKELY( ctx->failov_out_idx==ULONG_MAX || stem->cr_avail[ ctx->failov_out_idx ]<4UL ) ) return; + fd_failover_bus_msg_t * msg = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk ); + fd_memset( msg, 0, sizeof(*msg) ); + fd_failover_operator_t operator = { .epoch=ctx->operator_epoch }; + fd_memcpy( operator.identity, ctx->identity_pubkey, 32UL ); + fd_memcpy( operator.failover_identity, ctx->failover_pubkey, 32UL ); + fd_memcpy( msg->payload, &operator, sizeof(operator) ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_OPERATOR, ctx->failov_out_chunk, sizeof(*msg), 0UL, tspub, tspub ); + ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*msg), ctx->failov_out_chunk0, ctx->failov_out_wmark ); + ctx->operator_pending = 0; +} + static void FD_FN_SENSITIVE set_identity( fd_admin_tile_ctx_t * ctx, + fd_stem_context_t * stem, ulong slot_idx, void * data, ulong data_sz ) { @@ -636,14 +735,6 @@ set_identity( fd_admin_tile_ctx_t * ctx, FD_BASE58_ENCODE_32_BYTES( ctx->identity_pubkey, old_identity ); FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, "{\"old_identity\":\"%s\"}", old_identity ) ); - /* Under failover the failover tile moves the identity. */ - if( FD_UNLIKELY( ctx->failover_enabled ) ) { - FD_LOG_WARNING(( "set-identity is not supported while [failover.enabled] is true" )); - report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNSUPPORTED ); - fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNSUPPORTED ); - return; - } - if( FD_UNLIKELY( data_szfailover_enabled && fd_memeq( req->keypair+32UL, ctx->identity_pubkey, 32UL ) ) ) { + FD_LOG_NOTICE(( "set-identity: `%s` is already the identity, nothing to do", old_identity )); + report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS ); + fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_SUCCESS ); + return; + } + + if( FD_UNLIKELY( ctx->failover_enabled ) ) { + failover_identity_check_begin( ctx, req->keypair+32UL ); + ulong result; + while( (result = failover_identity_check_poll( ctx ))==ULONG_MAX ) FD_SPIN_PAUSE(); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + FD_LOG_WARNING(( "set-identity failed: the new failover identity is already an authorized voter" )); + report_admin_command_custom_result( &event, "authorized_voter" ); + fd_adminctl_complete( adminctl, slot_idx, result ); + return; + } + failover_operator_begin( ctx, req->keypair+32UL ); + } + + if( ctx->failov_out_idx!=ULONG_MAX ) find_identity_keyswitch( ctx, "tower" )->operator = 1UL; + ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; ulong identity_outset = (ulong)fd_log_wallclock(); for(;;) { @@ -691,6 +806,7 @@ set_identity( fd_admin_tile_ctx_t * ctx, } memcpy( ctx->identity_pubkey, req->keypair+32UL, 32UL ); + if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_operator_notify( ctx, stem ); report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS ); fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_SUCCESS ); @@ -1281,7 +1397,7 @@ failover_complete( fd_admin_tile_ctx_t * ctx, void const * resp, ulong resp_sz ) { fd_event_admin_command_t event = { - .type = FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER, + .type = ctx->failover_slot_cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_STATUS : FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE, .args_json = { '{', '}' }, .args_json_len = 2UL, .start_time = ctx->failover_start_time, @@ -1300,9 +1416,9 @@ failover_complete( fd_admin_tile_ctx_t * ctx, fd_adminctl_complete_response( ctx->adminctl, ctx->failover_slot_idx, result, resp, resp_sz ); if( FD_LIKELY( ctx->failover_cmd_cstr[ 0 ] ) ) { if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) { - FD_LOG_NOTICE(( "`failover %s` accepted by the failover tile, follow it with `failover status`", ctx->failover_cmd_cstr )); + FD_LOG_NOTICE(( "`%s` accepted by the failover tile, follow it with `failover status`", ctx->failover_cmd_cstr )); } else if( FD_UNLIKELY( result!=FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE ) ) { - FD_LOG_WARNING(( "`failover %s` refused by the failover tile (%s), nothing was done", ctx->failover_cmd_cstr, failover_result_name( result ) )); + FD_LOG_WARNING(( "`%s` refused by the failover tile (%s), nothing was done", ctx->failover_cmd_cstr, failover_result_name( result ) )); } } ctx->failover_slot_idx = ULONG_MAX; @@ -1312,15 +1428,55 @@ failover_complete( fd_admin_tile_ctx_t * ctx, ctx->failover_cmd_cstr[ 0 ] = '\0'; } -static void +/* Load a client's first target without changing its installed identity. + Repeating the same target leaves the controller and its history intact. + Admin serializes this with set-identity and authorized-voter changes. */ +static ulong FD_FN_SENSITIVE +failover_prepare( fd_admin_tile_ctx_t * ctx, + uchar const * keypair ) { + uchar pubkey[ 32 ]; + fd_ed25519_public_from_private( pubkey, keypair, ctx->sha512 ); + if( FD_UNLIKELY( !fd_memeq( pubkey, keypair+32UL, 32UL ) ) ) return FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH; + if( FD_UNLIKELY( fd_memeq( pubkey, ctx->junk_pubkey, 32UL ) ) ) return FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT; + if( ctx->failover_enabled ) return fd_memeq( pubkey, ctx->failover_pubkey, 32UL ) + ? FD_ADMINCTL_RESULT_SUCCESS : FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT; + if( FD_UNLIKELY( !fd_memeq( ctx->identity_pubkey, ctx->junk_pubkey, 32UL ) ) ) return FD_FAILOVER_CONTROL_RESULT_NOT_BOOT_IDENTITY; + + failover_identity_check_begin( ctx, pubkey ); + ulong result; + while( (result = failover_identity_check_poll( ctx ))==ULONG_MAX ) FD_SPIN_PAUSE(); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) return result; + + for( ulong i=0UL; itopo->tile_cnt; i++ ) { + fd_topo_tile_t const * tile = &ctx->topo->tiles[ i ]; + if( strcmp( tile->name, "sign" ) ) continue; + fd_keyswitch_t * sign = fd_topo_obj_laddr( ctx->topo, tile->id_keyswitch_obj_id ); + fd_memcpy( sign->bytes, keypair, 64UL ); + sign->param = FD_KEYSWITCH_PARAM_IDENTITY_PREPARE; + fd_keyswitch_state( sign, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + } + while( (result = failover_identity_check_poll( ctx ))==ULONG_MAX ) FD_SPIN_PAUSE(); + /* Preflight checked every sign tile before any target was loaded. */ + FD_TEST( result==FD_ADMINCTL_RESULT_SUCCESS ); + fd_memcpy( ctx->failover_pubkey, pubkey, 32UL ); + ctx->failover_enabled = 1; + ctx->operator_epoch++; + ctx->operator_pending = 1; + return FD_ADMINCTL_RESULT_SUCCESS; +} + +static void FD_FN_SENSITIVE failover_request( fd_admin_tile_ctx_t * ctx, fd_stem_context_t * stem, ulong slot_idx, - void const * data, + ulong cmd_id, + void * data, ulong data_sz ) { - fd_adminctl_t * adminctl = ctx->adminctl; - fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER, data, data_sz ); + int status = cmd_id==FD_ADMINCTL_CMD_FAILOVER_STATUS; + int event_type = status ? FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_STATUS : FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE; + fd_adminctl_t * adminctl = ctx->adminctl; + fd_event_admin_command_t event = prepare_admin_command( event_type, data, data_sz ); if( FD_UNLIKELY( data_sz=FD_ADMINCTL_FAILOVER_CMD_CNT ) invalid = "unknown command"; - else if( req.flags & ~(FD_ADMINCTL_FAILOVER_FLAG_YES|FD_ADMINCTL_FAILOVER_FLAG_FORCE) ) invalid = "unknown flags"; - else if( (req.cmd==FD_ADMINCTL_FAILOVER_CMD_PROMOTE)!=force ) invalid = "--force goes with promote, and a promote without it is sent as a handoff"; - else if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS && req.flags ) invalid = "status takes no flags"; - else if( req.cmd!=FD_ADMINCTL_FAILOVER_CMD_HANDOFF && (req.addr || req.port) ) invalid = "an address or port goes only with a handoff"; - else if( req.reserved[ 0 ] || req.reserved[ 1 ] ) invalid = "reserved bytes are set"; + if( req.cmd>=FD_ADMINCTL_FAILOVER_CMD_CNT ) invalid = "unknown command"; + else if( (req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS)!=status ) invalid = "payload command does not match the admin command"; + else if( req.flags & ~(FD_ADMINCTL_FAILOVER_FLAG_YES|FD_ADMINCTL_FAILOVER_FLAG_FORCE) ) invalid = "unknown flags"; + else if( (req.cmd==FD_ADMINCTL_FAILOVER_CMD_PROMOTE)!=force ) invalid = "--force goes with promote, and a promote without it is sent as a handoff"; + else if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS && req.flags ) invalid = "failover status takes no flags"; + else if( req.cmd!=FD_ADMINCTL_FAILOVER_CMD_HANDOFF && (req.addr || req.port) ) invalid = "an address or port goes only with a handoff"; + else if( req.reserved[ 0 ] || req.reserved[ 1 ] ) invalid = "reserved bytes are set"; if( FD_UNLIKELY( invalid ) ) { FD_LOG_WARNING(( "refusing adminctl failover cmd %lu flags %lu, %s", req.cmd, req.flags, invalid )); report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNKNOWN_COMMAND ); fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ); return; } - int status = req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS; - if( status ) { - FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, - "{\"command\":\"status\"}" ) ); - } else { + if( !status ) { FD_TEST( fd_cstr_printf_check( (char *)event.args_json, sizeof(event.args_json), &event.args_json_len, - "{\"command\":\"%s\",\"yes\":%s,\"force\":%s}", fd_adminctl_failover_cmd_name( req.cmd ), + "{\"yes\":%s,\"force\":%s}", (req.flags & FD_ADMINCTL_FAILOVER_FLAG_YES) ? "true" : "false", force ? "true" : "false" ) ); } @@ -1376,29 +1530,48 @@ failover_request( fd_admin_tile_ctx_t * ctx, force ? " --force" : "" ) ); if( FD_UNLIKELY( ctx->failov_out_idx==ULONG_MAX ) ) { - FD_LOG_WARNING(( "`failover %s` refused, the admin tile has no failover bus link", cmd_cstr )); + FD_LOG_WARNING(( "`%s` refused, the admin tile has no failover bus link", cmd_cstr )); report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNSUPPORTED ); fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNSUPPORTED ); return; } - /* One command at a time. A command stays outstanding until the - failover tile responds, even after its deadline, so commands cannot - pile up behind a stalled failover tile. Status then reports the - failover tile unresponsive rather than busy. */ - if( FD_UNLIKELY( ctx->failover_answered!=ctx->failover_nonce ) ) { - if( FD_UNLIKELY( status && ctx->failover_slot_idx==ULONG_MAX ) ) { + /* Keep one command outstanding until the failover tile responds, even + after timeout. Status reports a stalled tile as unresponsive. Publish + the last set-identity OPERATOR first so the command uses its identity. */ + if( FD_UNLIKELY( ctx->failover_answered!=ctx->failover_nonce || ctx->operator_pending ) ) { + if( FD_UNLIKELY( status && ( ctx->failover_slot_idx==ULONG_MAX || ctx->operator_pending ) ) ) { FD_LOG_WARNING(( "`failover status` refused, the failover tile has not responded to an earlier command, check the log" )); report_admin_command_custom_result( &event, "unresponsive" ); fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE ); return; } - FD_LOG_WARNING(( "`failover %s` refused, another failover command is still waiting on the failover tile, try again", cmd_cstr )); + FD_LOG_WARNING(( "`%s` refused, another failover command is still waiting on the failover tile, try again", cmd_cstr )); report_admin_command_custom_result( &event, "busy" ); fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_BUSY ); return; } + if( prepare ) { + if( FD_UNLIKELY( stem->cr_avail[ ctx->failov_out_idx ]<4UL ) ) { + report_admin_command_custom_result( &event, "busy" ); + fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_BUSY ); + return; + } + fd_adminctl_failover_prepare_t * payload = data; + ulong result = failover_prepare( ctx, payload->keypair ); + fd_memzero_explicit( payload->keypair, sizeof(payload->keypair) ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + report_admin_command_custom_result( &event, "target_refused" ); + fd_adminctl_complete( adminctl, slot_idx, result ); + return; + } + /* The public target and epoch must reach the controller before the + command. after_credit reserves enough credits for both frames. */ + failover_operator_notify( ctx, stem ); + FD_TEST( !ctx->operator_pending ); + } + /* Kept for the event and the log of the response. */ FD_TEST( event.args_json_len<=sizeof(ctx->failover_args_json) ); fd_memcpy( ctx->failover_args_json, event.args_json, event.args_json_len ); @@ -1406,7 +1579,7 @@ failover_request( fd_admin_tile_ctx_t * ctx, if( status ) ctx->failover_cmd_cstr[ 0 ] = '\0'; else { fd_cstr_ncpy( ctx->failover_cmd_cstr, cmd_cstr, sizeof(ctx->failover_cmd_cstr) ); - FD_LOG_NOTICE(( "forwarding `failover %s` to the failover tile", cmd_cstr )); + FD_LOG_NOTICE(( "forwarding `%s` to the failover tile", cmd_cstr )); } fd_failover_bus_msg_t * msg = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk ); @@ -1441,7 +1614,8 @@ failover_response( fd_admin_tile_ctx_t * ctx ) { } /* The failover tile asked for an identity switch. We only pass the - public key on, the sign tile selects the keypair it loaded at boot. + public key on, the sign tile selects the junk key or the failover + identity. Like set-identity this blocks until every tile has switched. We poll no adminctl command meanwhile, so `failover status` and every failover command wait for it. */ @@ -1456,7 +1630,13 @@ failover_switch_request( fd_admin_tile_ctx_t * ctx, fd_failover_switch_resp_t response; fd_memset( &response, 0, sizeof(response) ); response.result = FD_FAILOVER_SWITCH_ERR_DISABLED; - if( FD_LIKELY( ctx->failover_enabled ) ) { + if( FD_UNLIKELY( ctx->failover_enabled && req.epoch!=ctx->operator_epoch ) ) { + FD_LOG_WARNING(( "refusing the failover tile's identity switch, set-identity ran since it asked" )); + response.result = FD_FAILOVER_SWITCH_ERR_STALE; + response.operator.epoch = ctx->operator_epoch; + fd_memcpy( response.operator.identity, ctx->identity_pubkey, 32UL ); + fd_memcpy( response.operator.failover_identity, ctx->failover_pubkey, 32UL ); + } else if( FD_LIKELY( ctx->failover_enabled ) ) { fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_SET_IDENTITY, NULL, 0UL ); FD_BASE58_ENCODE_32_BYTES( ctx->identity_pubkey, old_identity ); FD_BASE58_ENCODE_32_BYTES( req.identity, new_identity ); @@ -1469,11 +1649,14 @@ failover_switch_request( fd_admin_tile_ctx_t * ctx, fd_memset( keypair, 0, 32UL ); fd_memcpy( keypair+32UL, req.identity, 32UL ); + find_identity_keyswitch( ctx, "tower" )->operator = 0UL; /* the tower keeps the failover rules */ + ctx->failover_switch = 1; ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; ulong identity_outset = (ulong)fd_log_wallclock(); for(;;) { if( FD_UNLIKELY( poll_set_identity( ctx, &state, identity_outset, keypair, NULL, 0UL ) ) ) break; } + ctx->failover_switch = 0; fd_memcpy( ctx->identity_pubkey, req.identity, 32UL ); /* The watermark is the tower's output sequence at its halt, which @@ -1518,9 +1701,11 @@ after_credit( fd_admin_tile_ctx_t * ctx, void * payload = NULL; ulong payload_sz = 0UL; + failover_operator_notify( ctx, stem ); + if( FD_UNLIKELY( ctx->failover_slot_idx!=ULONG_MAX && fd_tickcount()>ctx->failover_deadline && !failov_frag_waiting( ctx, stem ) ) ) { - if( FD_LIKELY( ctx->failover_cmd_cstr[ 0 ] ) ) FD_LOG_WARNING(( "the failover tile did not answer `failover %s` in time, it may still run, check `failover status` and the log", ctx->failover_cmd_cstr )); + if( FD_LIKELY( ctx->failover_cmd_cstr[ 0 ] ) ) FD_LOG_WARNING(( "the failover tile did not answer `%s` in time, it may still run, check `failover status` and the log", ctx->failover_cmd_cstr )); else FD_LOG_WARNING(( "the failover tile did not answer `failover status` in time, check the log" )); failover_complete( ctx, FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE, NULL, 0UL ); *charge_busy = 1; @@ -1539,8 +1724,9 @@ after_credit( fd_admin_tile_ctx_t * ctx, *charge_busy = 1; break; case FD_ADMINCTL_CMD_SET_IDENTITY: - set_identity( ctx, slot_idx, payload, payload_sz ); + set_identity( ctx, stem, slot_idx, payload, payload_sz ); *charge_busy = 1; + if( FD_UNLIKELY( ctx->failover_enabled ) ) *opt_poll_in = 0; /* it may have published OPERATOR */ break; case FD_ADMINCTL_CMD_REMOVE_ALL_AUTH_VOTERS: remove_all_authorized_voters( ctx, slot_idx, payload, payload_sz ); @@ -1556,9 +1742,10 @@ after_credit( fd_admin_tile_ctx_t * ctx, *charge_busy = 1; *opt_poll_in = 0; break; - case FD_ADMINCTL_CMD_FAILOVER: - if( FD_UNLIKELY( ctx->failover_enabled ) ) { - failover_request( ctx, stem, slot_idx, payload, payload_sz ); + case FD_ADMINCTL_CMD_FAILOVER_PROMOTE: + case FD_ADMINCTL_CMD_FAILOVER_STATUS: + if( FD_UNLIKELY( ctx->failov_out_idx!=ULONG_MAX ) ) { + failover_request( ctx, stem, slot_idx, cmd_id, payload, payload_sz ); *charge_busy = 1; *opt_poll_in = 0; break; @@ -1641,7 +1828,7 @@ populate_allowed_fds( fd_topo_t const * topo FD_PARAM_UNUSED, return out_cnt; } -#define STEM_BURST (1UL) +#define STEM_BURST (2UL) #define STEM_LAZY ((long)1e6) /* 1ms */ #define STEM_ALWAYS_PARK 1 diff --git a/src/discof/admin/fd_adminctl.h b/src/discof/admin/fd_adminctl.h index cf6f0db53ae..d0e1ece8c78 100644 --- a/src/discof/admin/fd_adminctl.h +++ b/src/discof/admin/fd_adminctl.h @@ -37,7 +37,8 @@ #define FD_ADMINCTL_CMD_GET_IDENTITY (3UL) #define FD_ADMINCTL_CMD_REMOVE_ALL_AUTH_VOTERS (4UL) #define FD_ADMINCTL_CMD_SNAP_CREATE (5UL) -#define FD_ADMINCTL_CMD_FAILOVER (6UL) +#define FD_ADMINCTL_CMD_FAILOVER_PROMOTE (6UL) +#define FD_ADMINCTL_CMD_FAILOVER_STATUS (7UL) #define FD_ADMINCTL_ALIGN (8UL) #define FD_ADMINCTL_PAYLOAD_MAX (32768UL) @@ -64,20 +65,24 @@ #define FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH (0x3001UL) #define FD_SET_IDENTITY_RESULT_INVALID_VOTE_HISTORY (0x3002UL) #define FD_SET_IDENTITY_RESULT_SAME_IDENTITY (0x3003UL) +#define FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER (0x3004UL) #define FD_FAILOVER_CONTROL_RESULT_BUSY (0x4001UL) /* another failover command is waiting on the failover tile */ #define FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE (0x4002UL) /* the failover tile did not respond in time */ -#define FD_FAILOVER_CONTROL_RESULT_BAD_ROLE (0x4003UL) /* promote on the active, demote on a standby */ +#define FD_FAILOVER_CONTROL_RESULT_BAD_ROLE (0x4003UL) /* promote on the active */ #define FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS (0x4004UL) /* a transition or key switch is running */ #define FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY (0x4006UL) /* the bound peer cannot complete this handoff */ #define FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE (0x4007UL) /* the authenticated peer holds the identity */ #define FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING (0x4008UL) /* the peer has not responded to our handoff */ -#define FD_FAILOVER_CONTROL_RESULT_TAKEN (0x4009UL) /* the peer took our handoff, only a new local tenure clears this */ +#define FD_FAILOVER_CONTROL_RESULT_TAKEN (0x4009UL) /* the peer took our handoff, cleared once we hold the identity again or set-identity restarts failover */ #define FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN (0x400AUL) /* gossip has a fresh contact info for the staked identity from another host */ #define FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER (0x400CUL) /* active has no eligible final vote state, see its local voting diagnostics */ #define FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS (0x400EUL) /* gossip has no address for the active */ #define FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED (0x400FUL) /* no check found another holder, which does not prove the peer cannot sign */ #define FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND (0x4010UL) /* this standby's replay is too far behind the active's last vote */ +#define FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED (0x4011UL) /* client has no target identity */ +#define FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT (0x4012UL) /* target differs from the prepared identity or is the boot identity */ +#define FD_FAILOVER_CONTROL_RESULT_NOT_BOOT_IDENTITY (0x4013UL) /* client must prepare while still on its boot identity */ struct fd_adminctl_add_auth_voter_v1 { ulong version; /* ==FD_ADMINCTL_ADD_AUTH_VOTER_PAYLOAD_VERSION */ @@ -124,26 +129,19 @@ typedef struct fd_adminctl_remove_all_auth_voters_v1 fd_adminctl_remove_all_auth /* Failover commands, forwarded to the failover tile. */ #define FD_ADMINCTL_FAILOVER_CMD_HANDOFF (0UL) -#define FD_ADMINCTL_FAILOVER_CMD_DEMOTE (1UL) -#define FD_ADMINCTL_FAILOVER_CMD_PROMOTE (2UL) -#define FD_ADMINCTL_FAILOVER_CMD_STATUS (3UL) -#define FD_ADMINCTL_FAILOVER_CMD_CNT (4UL) - -static inline char const * -fd_adminctl_failover_cmd_name( ulong cmd ) { - switch( cmd ) { - case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: return "handoff"; - case FD_ADMINCTL_FAILOVER_CMD_DEMOTE: return "demote"; - case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: return "promote"; - case FD_ADMINCTL_FAILOVER_CMD_STATUS: return "status"; - default: return "unknown"; - } -} +#define FD_ADMINCTL_FAILOVER_CMD_PROMOTE (1UL) +#define FD_ADMINCTL_FAILOVER_CMD_STATUS (2UL) +#define FD_ADMINCTL_FAILOVER_CMD_CNT (3UL) /* The command as the operator typed it. */ static inline char const * fd_adminctl_failover_cmd_typed( ulong cmd ) { - return cmd==FD_ADMINCTL_FAILOVER_CMD_HANDOFF ? "promote" : fd_adminctl_failover_cmd_name( cmd ); + switch( cmd ) { + case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: + case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: return "failover promote"; + case FD_ADMINCTL_FAILOVER_CMD_STATUS: return "failover status"; + default: return "unknown"; + } } #define FD_ADMINCTL_FAILOVER_FLAG_YES (1UL) /* confirmation only, does not authorize peer or history overrides */ @@ -159,12 +157,20 @@ struct fd_adminctl_failover_req_v1 { }; typedef struct fd_adminctl_failover_req_v1 fd_adminctl_failover_req_t; +/* Optional promote payload. Admin consumes and wipes the keypair. + Only req may be forwarded to the failover tile. */ +struct fd_adminctl_failover_prepare_v1 { + fd_adminctl_failover_req_t req; + uchar keypair[ 64 ]; +}; +typedef struct fd_adminctl_failover_prepare_v1 fd_adminctl_failover_prepare_t; + struct fd_adminctl_failover_control_resp_v1 { ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ uchar role; /* FD_FAILOVER_ROLE_* after the command */ uchar action; /* controller action after the command */ uchar reserved[ 6 ]; - ulong handoff_id; /* the handoff an accepted promote or demote opened or resumed, 0 none */ + ulong handoff_id; /* the handoff an accepted promote opened or resumed, 0 none */ }; typedef struct fd_adminctl_failover_control_resp_v1 fd_adminctl_failover_control_resp_t; #define FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION (1UL) @@ -202,7 +208,7 @@ FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)==24UL, failover_co /* `failover status`, provided by the failover tile. */ struct fd_adminctl_failover_status_resp_v1 { ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ - uchar enabled; /* 1 from the failover tile */ + uchar enabled; /* 1 accepts handoffs, 2 is an outgoing client only */ uchar role; /* FD_FAILOVER_ROLE_* */ uchar action; /* FD_FAILOVER_ACTION_* */ uchar stuck; /* a transition failed or is overdue */ diff --git a/src/discof/admin/test_admin_tile.c b/src/discof/admin/test_admin_tile.c index f1a59788c37..1c47a8bf0b8 100644 --- a/src/discof/admin/test_admin_tile.c +++ b/src/discof/admin/test_admin_tile.c @@ -193,6 +193,183 @@ test_set_identity( void ) { FD_TEST( state==FD_SET_IDENTITY_STATE_UNLOCKED && replay_ks->state==FD_KEYSWITCH_STATE_UNLOCKED ); } +static uchar adminctl_mem[ 1UL<<18 ] __attribute__((aligned(FD_ADMINCTL_ALIGN))); + +/* Under failover, set-identity changes the failover identity unless the + key is our junk key. OPERATOR waits while the link has no room and + failover commands are busy until it goes out. A switch request from + before set-identity gets STALE with the failover identity. */ + +static fd_frag_meta_t bus_mcache[ 8 ]; +static ulong bus_seq; +static ulong bus_depth = 8UL; +static ulong bus_cr_avail; +static ulong bus_min_cr_avail; +static int bus_reliable = 1; +static fd_frag_meta_t * bus_mcaches[ 1 ] = { bus_mcache }; +static fd_stem_context_t bus_stem[ 1 ]; +static uchar bus_out[ 1024 ] __attribute__((aligned(128))); + +static void +bus_setup( ulong credits ) { + fd_memset( bus_mcache, 0, sizeof(bus_mcache) ); + bus_seq = 0UL; + bus_cr_avail = credits; + bus_min_cr_avail = credits; + *bus_stem = (fd_stem_context_t){ .mcaches=bus_mcaches, .seqs=&bus_seq, .depths=&bus_depth, + .cr_avail=&bus_cr_avail, .min_cr_avail=&bus_min_cr_avail, + .cr_decrement_amount=1UL, .out_reliable=&bus_reliable }; + ctx.failov_out_idx = 0UL; + ctx.failov_out_mem = (fd_wksp_t *)bus_out; + ctx.failov_out_chunk0 = ctx.failov_out_wmark = ctx.failov_out_chunk = 0UL; +} + +static fd_failover_operator_t +bus_operator( void ) { + FD_TEST( bus_seq && bus_mcache[ bus_seq-1UL ].sig==FD_FAILOVER_BUS_OPERATOR ); + fd_failover_operator_t operator; + fd_memcpy( &operator, ((fd_failover_bus_msg_t const *)bus_out)->payload, sizeof(operator) ); + return operator; +} + +/* Events identify the public command on completion and on rejection, + including requests whose payload cannot be decoded. */ +static void +test_failover_command( ulong cmd_id, + ulong payload_cmd, + ulong flags, + int event_type ) { + setup(); + bus_setup( 8UL ); + ctx.adminctl = fd_adminctl_join( fd_adminctl_new( adminctl_mem ) ); + ctx.failover_enabled = 1; + ctx.failover_slot_idx = ULONG_MAX; + FD_TEST( ctx.adminctl ); + + static fd_frag_meta_t event_mcache[ 128 ] __attribute__((aligned(128))); + static uchar event_mem[ 1024 ] __attribute__((aligned(128))); + ulong event_seq = 0UL; + fd_event_reporter_t reporter = { .mcache=event_mcache, .depth=128UL, .seq_store=&event_seq, + .mem=(fd_wksp_t *)event_mem, .mtu=sizeof(event_mem) }; + fd_event_tl = &reporter; + + for( ulong i=0UL; i<4UL; i++ ) { + void * payload; + ulong payload_max; + ulong slot_idx = fd_adminctl_reserve( ctx.adminctl, &payload, &payload_max ); + FD_TEST( slot_idx!=ULONG_MAX && payload_max>=sizeof(fd_adminctl_failover_req_t) ); + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=payload_cmd, .flags=flags }; + /* A status command must not carry a promotion, or vice versa. */ + if( i==3UL ) req.cmd = payload_cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? FD_ADMINCTL_FAILOVER_CMD_HANDOFF : FD_ADMINCTL_FAILOVER_CMD_STATUS; + fd_memcpy( payload, &req, sizeof(req) ); + fd_adminctl_publish( ctx.adminctl, slot_idx, cmd_id, i==2UL ? 0UL : sizeof(req) ); + int poll_in = 1; + int busy = 0; + ulong seq_before = bus_seq; + for( ulong j=0UL; jpayload, sizeof(forwarded) ); + FD_TEST( forwarded.cmd==payload_cmd && forwarded.flags==flags ); + result = i ? FD_FAILOVER_CONTROL_RESULT_BAD_ROLE : FD_ADMINCTL_RESULT_SUCCESS; + event_result = i ? FD_EVENT_ADMIN_COMMAND_RESULT_CUSTOM : FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS; + ctx.failov_in.nonce = ctx.failover_nonce; + ctx.failov_in.result = result; + failover_response( &ctx ); + } else { + FD_TEST( bus_seq==seq_before ); + result = i==2UL ? FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH : FD_ADMINCTL_RESULT_UNKNOWN_COMMAND; + event_result = i==2UL ? FD_EVENT_ADMIN_COMMAND_RESULT_ABI_SIZE_MISMATCH : FD_EVENT_ADMIN_COMMAND_RESULT_UNKNOWN_COMMAND; + } + FD_TEST( fd_adminctl_wait( ctx.adminctl, slot_idx )==result ); + FD_TEST( event_seq==i+1UL ); + fd_event_admin_command_t event; + fd_memcpy( &event, event_mem, sizeof(event) ); + FD_TEST( event.type==event_type && event.result==event_result ); + if( i<2UL ) { + char const * args = payload_cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? "{}" : + flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ? "{\"yes\":true,\"force\":true}" : "{\"yes\":false,\"force\":false}"; + FD_TEST( event.args_json_len==strlen( args ) && !memcmp( event.args_json, args, event.args_json_len ) ); + } + } + fd_event_tl = NULL; +} + +static void +test_failover_identity( void ) { + setup(); + FD_TEST( fd_topob_new( test_topo, "admin-test" ) ); + fd_topob_wksp( test_topo, "wksp" )->wksp = NULL; + fd_keyswitch_t * tower_ks = mock_tile( "tower" ); + fd_keyswitch_t * sign_ks = mock_tile( "sign" ); + ctx.topo = test_topo; + ctx.failover_enabled = 1; + fd_memset( ctx.junk_pubkey, 0x11, 32UL ); + fd_memset( ctx.failover_pubkey, 0x5A, 32UL ); + fd_memcpy( ctx.identity_pubkey, ctx.junk_pubkey, 32UL ); + uchar third[ 32 ]; fd_memset( third, 0x66, 32UL ); + bus_setup( 8UL ); + + /* A key other than junk and staked becomes the failover identity, and + nothing goes to the failover tile until every tile switched. */ + failover_operator_begin( &ctx, third ); + FD_TEST( ctx.operator_epoch==1UL && fd_memeq( ctx.failover_pubkey, third, 32UL ) && ctx.operator_pending && !bus_seq ); + FD_TEST( sign_ks->param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR && tower_ks->operator==1UL ); + fd_memcpy( ctx.identity_pubkey, third, 32UL ); + failover_operator_notify( &ctx, bus_stem ); + fd_failover_operator_t operator = bus_operator(); + FD_TEST( !ctx.operator_pending && operator.epoch==1UL ); + FD_TEST( fd_memeq( operator.identity, third, 32UL ) && fd_memeq( operator.failover_identity, third, 32UL ) ); + + /* The junk key keeps the failover identity. With three credits left + OPERATOR waits, and goes out once there is room. */ + bus_cr_avail = 3UL; + failover_operator_begin( &ctx, ctx.junk_pubkey ); + fd_memcpy( ctx.identity_pubkey, ctx.junk_pubkey, 32UL ); + failover_operator_notify( &ctx, bus_stem ); + FD_TEST( ctx.operator_pending && bus_seq==1UL && fd_memeq( ctx.failover_pubkey, third, 32UL ) ); + FD_TEST( fd_sha512_join( fd_sha512_new( ctx.sha512 ) ) ); + ctx.adminctl = fd_adminctl_join( fd_adminctl_new( adminctl_mem ) ); + FD_TEST( ctx.adminctl ); + void * payload; + ulong payload_max; + ulong slot_idx = fd_adminctl_reserve( ctx.adminctl, &payload, &payload_max ); + FD_TEST( slot_idx!=ULONG_MAX ); + fd_adminctl_failover_req_t creq = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_HANDOFF }; + fd_memcpy( payload, &creq, sizeof(creq) ); + fd_adminctl_publish( ctx.adminctl, slot_idx, FD_ADMINCTL_CMD_FAILOVER_PROMOTE, sizeof(creq) ); + ulong poll_idx; + void * data; + ulong data_sz; + FD_TEST( fd_adminctl_poll( ctx.adminctl, &poll_idx, &data, &data_sz )==FD_ADMINCTL_CMD_FAILOVER_PROMOTE ); + failover_request( &ctx, bus_stem, poll_idx, FD_ADMINCTL_CMD_FAILOVER_PROMOTE, data, data_sz ); + FD_TEST( bus_seq==1UL ); + FD_TEST( fd_adminctl_wait( ctx.adminctl, slot_idx )==FD_FAILOVER_CONTROL_RESULT_BUSY ); + bus_cr_avail = 8UL; + failover_operator_notify( &ctx, bus_stem ); + operator = bus_operator(); + FD_TEST( !ctx.operator_pending && operator.epoch==2UL ); + FD_TEST( fd_memeq( operator.identity, ctx.junk_pubkey, 32UL ) && fd_memeq( operator.failover_identity, third, 32UL ) ); + + /* A switch request from before set-identity gets STALE with what it + installed. */ + ctx.failov_in.nonce = 9UL; + fd_failover_switch_req_t sreq = { .epoch=1UL }; + fd_memcpy( sreq.identity, third, 32UL ); + fd_memcpy( ctx.failov_in.payload, &sreq, sizeof(sreq) ); + failover_switch_request( &ctx, bus_stem ); + fd_failover_switch_resp_t answer; + fd_memcpy( &answer, ((fd_failover_bus_msg_t const *)bus_out)->payload, sizeof(answer) ); + FD_TEST( answer.result==FD_FAILOVER_SWITCH_ERR_STALE && answer.operator.epoch==2UL ); + FD_TEST( fd_memeq( answer.operator.identity, ctx.junk_pubkey, 32UL ) && fd_memeq( answer.operator.failover_identity, third, 32UL ) ); + ctx.topo = NULL; +} + int main( int argc, char ** argv ) { @@ -201,6 +378,11 @@ main( int argc, test_add_authorized_voter(); test_remove_all_authorized_voters(); test_set_identity(); + test_failover_command( FD_ADMINCTL_CMD_FAILOVER_STATUS, FD_ADMINCTL_FAILOVER_CMD_STATUS, 0UL, FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_STATUS ); + test_failover_command( FD_ADMINCTL_CMD_FAILOVER_PROMOTE, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0UL, FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE ); + test_failover_command( FD_ADMINCTL_CMD_FAILOVER_PROMOTE, FD_ADMINCTL_FAILOVER_CMD_PROMOTE, + FD_ADMINCTL_FAILOVER_FLAG_YES|FD_ADMINCTL_FAILOVER_FLAG_FORCE, FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE ); + test_failover_identity(); FD_LOG_NOTICE(( "pass" )); fd_halt(); diff --git a/src/discof/failover/fd_failover_bus.h b/src/discof/failover/fd_failover_bus.h index 915b58f413b..c87a1a2a7bb 100644 --- a/src/discof/failover/fd_failover_bus.h +++ b/src/discof/failover/fd_failover_bus.h @@ -17,22 +17,40 @@ #define FD_FAILOVER_BUS_SWITCH_RESP (2UL) /* admin to failov, fd_failover_switch_resp_t */ #define FD_FAILOVER_BUS_REQUEST (3UL) /* admin to failov, fd_adminctl_failover_req_t */ #define FD_FAILOVER_BUS_RESPONSE (4UL) /* failov to admin, the requested command's response */ +#define FD_FAILOVER_BUS_OPERATOR (5UL) /* admin to failov, fd_failover_operator_t, set-identity ran */ -/* Only the public key goes over the bus. The sign tile only accepts - a keypair it loaded at boot. */ +/* Only the public key goes over the bus, and the sign tile selects its + junk key or the failover identity. epoch is the failover epoch the + failover tile last saw, and the admin tile refuses a request from + before a set-identity. */ struct fd_failover_switch_req { uchar identity[ 32 ]; /* public key of the identity to install */ + ulong epoch; }; typedef struct fd_failover_switch_req fd_failover_switch_req_t; #define FD_FAILOVER_SWITCH_OK (0UL) #define FD_FAILOVER_SWITCH_ERR_DISABLED (1UL) /* failover is off here */ +#define FD_FAILOVER_SWITCH_ERR_STALE (2UL) /* set-identity ran since the request, nothing switched */ -/* Sent once every tile has switched. */ +/* set-identity ran, and every set-identity bumps the failover epoch. + identity is the public key it installed, and failover_identity is the + key failover hands off, which set-identity changes to identity unless + that is the junk key. */ +struct fd_failover_operator { + ulong epoch; + uchar identity[ 32 ]; + uchar failover_identity[ 32 ]; +}; + +typedef struct fd_failover_operator fd_failover_operator_t; + +/* Sent once every tile has switched, or right away when refused. */ struct fd_failover_switch_resp { - ulong result; /* FD_FAILOVER_SWITCH_* */ - ulong tower_watermark; /* the tower keyswitch result, its output sequence at the halt */ + ulong result; /* FD_FAILOVER_SWITCH_* */ + ulong tower_watermark; /* the tower keyswitch result, its output sequence at the halt */ + fd_failover_operator_t operator; /* with STALE */ }; typedef struct fd_failover_switch_resp fd_failover_switch_resp_t; @@ -41,6 +59,7 @@ typedef struct fd_failover_switch_resp fd_failover_switch_resp_t; FD_STATIC_ASSERT( sizeof(fd_failover_switch_req_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, switch_req_fits ); FD_STATIC_ASSERT( sizeof(fd_failover_switch_resp_t)<=FD_FAILOVER_BUS_PAYLOAD_MAX, switch_resp_fits ); +FD_STATIC_ASSERT( sizeof(fd_failover_operator_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, operator_fits ); FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_req_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, command_fits ); FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)<=FD_FAILOVER_BUS_PAYLOAD_MAX, control_resp_fits ); FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_status_resp_t )<=FD_FAILOVER_BUS_PAYLOAD_MAX, status_resp_fits ); diff --git a/src/discof/failover/fd_failover_channel.c b/src/discof/failover/fd_failover_channel.c index 3abe87bf90c..054c3f725b6 100644 --- a/src/discof/failover/fd_failover_channel.c +++ b/src/discof/failover/fd_failover_channel.c @@ -95,7 +95,7 @@ fd_failover_channel_new( void * shmem ) { ch->backoff_min = FD_FAILOVER_CHANNEL_BACKOFF_MIN_NANOS; ch->backoff_max = FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS; ch->backoff = ch->backoff_min; - ch->rate_credit = 16UL*1000000000UL; + ch->rate_credit = 16UL*FD_FAILOVER_NANOS_PER_SECOND; FD_TEST( fd_sha512_join( fd_sha512_new( ch->sha ) ) ); ch->magic = FD_FAILOVER_CHANNEL_MAGIC; return shmem; @@ -319,7 +319,7 @@ lose( fd_failover_channel_t * ch, ulong cause, char const * why ) { if( ch->paired_idx==(int)idx && now>=ch->loss_log_at[cause] ) { - ch->loss_log_at[cause] = fd_long_sat_add( now, 60000000000L ); + ch->loss_log_at[cause] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_WARNING(( "lost the failover session with `" FD_IP4_ADDR_FMT "`, %s, repeated failures limited to one line per minute", FD_IP4_ADDR_FMT_ARGS( ch->candidates[idx].address ), why )); } @@ -350,8 +350,8 @@ refill( ulong credit, ulong burst ) { if( now>*updated ) { /* Bound elapsed time before multiplying the scaled token count. */ - ulong elapsed = fd_ulong_min( (ulong)now-(ulong)*updated, 1000000000UL ); - credit = fd_ulong_min( credit+elapsed*rate, burst*1000000000UL ); + ulong elapsed = fd_ulong_min( (ulong)now-(ulong)*updated, FD_FAILOVER_NANOS_PER_SECOND ); + credit = fd_ulong_min( credit+elapsed*rate, burst*FD_FAILOVER_NANOS_PER_SECOND ); *updated = now; } return credit; @@ -362,7 +362,7 @@ admit( fd_failover_channel_t * ch, uint address, long now ) { ch->rate_credit = refill( ch->rate_credit, &ch->rate_updated, now, 32UL, 16UL ); - if( ch->rate_credit<1000000000UL ) return 0; + if( ch->rate_creditcandidates[i].fd!=-1 && ch->candidates[i].address==address; @@ -373,16 +373,16 @@ admit( fd_failover_channel_t * ch, struct source * s = &ch->sources[i]; if( s->used ) s->credit = refill( s->credit, &s->updated, now, 4UL, 2UL ); if( s->used && s->address==address ) { source = s; break; } - if( !s->used || s->credit==2000000000UL ) reusable = s; + if( !s->used || s->credit==2UL*FD_FAILOVER_NANOS_PER_SECOND ) reusable = s; } if( !source ) { if( !reusable ) return 0; source = reusable; - *source = (struct source){ .address=address, .used=1, .updated=now, .credit=2000000000UL }; + *source = (struct source){ .address=address, .used=1, .updated=now, .credit=2UL*FD_FAILOVER_NANOS_PER_SECOND }; } - if( source->credit<1000000000UL ) return 0; - source->credit -= 1000000000UL; - ch->rate_credit -= 1000000000UL; + if( source->creditcredit -= FD_FAILOVER_NANOS_PER_SECOND; + ch->rate_credit -= FD_FAILOVER_NANOS_PER_SECOND; return 1; } @@ -526,15 +526,15 @@ hello_err_name( int err ) { switch( err ) { case HELLO_ERR_TLS_KEY: return "its junk key is not the key TLS authenticated, check who can reach the failover port"; case FD_FAILOVER_HELLO_ERR_VERSION: return "the protocol version differs, run the same Firedancer version on both machines"; - case FD_FAILOVER_HELLO_ERR_STAKED: return "the staked identity differs, check that [paths.identity_key] is the same keypair on both machines"; + case FD_FAILOVER_HELLO_ERR_STAKED: return "the failover identity differs, check that both machines run [paths.identity_key] with the same keypair, or the same key from their last `set-identity`"; case FD_FAILOVER_HELLO_ERR_VOTE_ACCT: return "the vote account differs, check that [paths.vote_account] is the same on both machines"; case FD_FAILOVER_HELLO_ERR_JUNK_EQ: return "its junk key is ours, check that the address is the other machine, and if both machines share one [paths.base]/junk-identity.json delete it on one of them and restart that machine, never copy that file"; case FD_FAILOVER_HELLO_ERR_JUNK_STAKE: return "a junk key is the staked key, [paths.base]/junk-identity.json must not be the staked [paths.identity_key]"; - case FD_FAILOVER_HELLO_ERR_BOTH_ACT: return "both members are active, only one machine may run the staked identity, stop signing now on the machine that should not vote with `failover demote` there"; + case FD_FAILOVER_HELLO_ERR_BOTH_ACT: return "both members are active, only one machine may run the staked identity, stop signing now on the machine that should not vote with `set-identity` and its junk identity there"; case FD_FAILOVER_HELLO_ERR_ROLE: return "the role is unknown, run the same Firedancer version on both machines"; case FD_FAILOVER_HELLO_ERR_BOOT_ID: return "the boot id is zero, run the same Firedancer version on both machines"; case FD_FAILOVER_HELLO_ERR_MODE: return "the consensus mode differs, run both machines in the same consensus mode"; - case FD_FAILOVER_HELLO_ERR_CERT: return "its member certificate is not signed by our staked key, check that [paths.identity_key] is the same keypair on both machines"; + case FD_FAILOVER_HELLO_ERR_CERT: return "its member certificate is not signed by the failover identity, check that both machines hold the same keypair for it"; default: return "unknown"; } } @@ -557,7 +557,7 @@ read_frame( fd_failover_channel_t * ch, /* The listener responds only to a HELLO it accepts, so a close here usually means it refused ours. */ if( FD_UNLIKELY( c->dialed && c->phase==PHASE_HELLO && now>=ch->dial_log_at[1] ) ) { - ch->dial_log_at[1] = fd_long_sat_add( now, 60000000000L ); + ch->dial_log_at[1] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` closed the connection before answering our HELLO, its log says why it refused us", FD_IP4_ADDR_FMT_ARGS( c->address ) )); } @@ -589,7 +589,7 @@ connect_candidate( fd_failover_channel_t * ch, if( connect( c->fd, fd_type_pun( &addr ), sizeof(addr) ) && errno!=EISCONN ) { if( errno==EINPROGRESS || errno==EALREADY || errno==EINTR ) return 0; /* TCP is still connecting. */ if( now>=ch->dial_log_at[0] ) { - ch->dial_log_at[0] = fd_long_sat_add( now, 60000000000L ); + ch->dial_log_at[0] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_WARNING(( "could not connect to the failover peer at `" FD_IP4_ADDR_FMT ":%hu` (%i-%s), check that it runs and that the machines can reach each other on [failover.listen_port]", FD_IP4_ADDR_FMT_ARGS( ch->peer_addr ), ch->peer_port, errno, fd_io_strerror( errno ) )); } @@ -614,7 +614,7 @@ handshake_candidate( fd_failover_channel_t * ch, uint reason = c->tls.conn.hs.base.reason; ulong kind = tls_log_kind( reason ); if( now>=ch->tls_log_at[kind] ) { - ch->tls_log_at[kind] = fd_long_sat_add( now, 60000000000L ); + ch->tls_log_at[kind] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_WARNING(( "TLS handshake with `" FD_IP4_ADDR_FMT "` failed (%s), not necessarily the other failover machine, check peer reachability and junk keys on both machines", FD_IP4_ADDR_FMT_ARGS( c->address ), reason ? fd_tls_reason_cstr( reason ) : "connection closed or socket error" )); } @@ -645,7 +645,7 @@ read_candidate_hello( fd_failover_channel_t * ch, if( read_frame( ch, idx, now, busy, &type, payload, &sz )<=0 ) return 0; /* Wait for HELLO, or stop after a read failure. */ if( type!=FD_FAILOVER_MSG_HELLO || sz!=sizeof(c->hello) ) { if( now>=ch->hello_log_at[0] ) { - ch->hello_log_at[0] = fd_long_sat_add( now, 60000000000L ); + ch->hello_log_at[0] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_WARNING(( "rejected the failover HELLO from `" FD_IP4_ADDR_FMT "`, its first frame is not a HELLO, check who can reach the failover port", FD_IP4_ADDR_FMT_ARGS( c->address ) )); } /* A malformed HELLO closes this candidate without pairing. */ @@ -660,7 +660,7 @@ read_candidate_hello( fd_failover_channel_t * ch, if( err!=FD_FAILOVER_HELLO_OK ) { ulong bucket = err==HELLO_ERR_TLS_KEY ? 12UL : ( err>0 && err<=FD_FAILOVER_HELLO_ERR_CERT ? (ulong)err : 13UL ); if( now>=ch->hello_log_at[bucket] ) { - ch->hello_log_at[bucket] = fd_long_sat_add( now, 60000000000L ); + ch->hello_log_at[bucket] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_WARNING(( "rejected the failover HELLO from `" FD_IP4_ADDR_FMT "`, %s", FD_IP4_ADDR_FMT_ARGS( c->address ), hello_err_name( err ) )); } /* A refused HELLO closes this candidate without pairing. */ @@ -702,7 +702,7 @@ pair_candidate( fd_failover_channel_t * ch, ch->expected_close = 0; *busy = 1; if( now>=ch->pair_log_at ) { - ch->pair_log_at = fd_long_sat_add( now, 60000000000L ); + ch->pair_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); char member[ FD_BASE58_ENCODED_32_SZ ]; fd_base58_encode_32( c->hello.junk_pubkey, NULL, member ); FD_LOG_NOTICE(( "paired with the failover peer at `" FD_IP4_ADDR_FMT "` via %s, authenticated member %s, role %s, boot id %016lx", @@ -765,7 +765,7 @@ fd_failover_channel_poll( fd_failover_channel_t * ch, struct candidate * c = &ch->candidates[i]; if( c->fd!=-1 && (int)i!=ch->paired_idx && now>=c->deadline ) { if( c->dialed && now>=ch->dial_log_at[2] ) { - ch->dial_log_at[2] = fd_long_sat_add( now, 60000000000L ); + ch->dial_log_at[2] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` did not finish the handshake in time, check that it runs and that the machines can reach each other on [failover.listen_port]", FD_IP4_ADDR_FMT_ARGS( c->address ) )); } diff --git a/src/discof/failover/fd_failover_channel.h b/src/discof/failover/fd_failover_channel.h index b0c7eb66819..704699e4552 100644 --- a/src/discof/failover/fd_failover_channel.h +++ b/src/discof/failover/fd_failover_channel.h @@ -11,13 +11,16 @@ #define FD_FAILOVER_CHANNEL_MAGIC (0xF17EDA2CE5FA170AUL) #define FD_FAILOVER_CHANNEL_CANDIDATE_MAX (16UL) +#define FD_FAILOVER_NANOS_PER_SECOND (1000000000UL) +#define FD_FAILOVER_LOG_INTERVAL_NANOS (60L*(long)FD_FAILOVER_NANOS_PER_SECOND) + /* fd_failover_clock returns monotonic nanoseconds for every channel deadline. */ static inline long fd_failover_clock( void ) { struct timespec ts; clock_gettime( CLOCK_MONOTONIC, &ts ); - return (long)ts.tv_sec*1000000000L + (long)ts.tv_nsec; + return (long)ts.tv_sec*(long)FD_FAILOVER_NANOS_PER_SECOND + (long)ts.tv_nsec; } /* Handshake, operation and retry limits in nanoseconds. */ diff --git a/src/discof/failover/fd_failover_tile.c b/src/discof/failover/fd_failover_tile.c index b1063f07936..c05a7ea3c5b 100644 --- a/src/discof/failover/fd_failover_tile.c +++ b/src/discof/failover/fd_failover_tile.c @@ -20,11 +20,11 @@ #include "generated/fd_failover_tile_seccomp.h" -/* The failov tile owns the socket to the other failover machine and is - the only tile that keeps host networking for it. It holds no private - key. The sign tile signs our member certificate with the staked key - and our TLS handshakes with the junk key. The admin tile keeps the - identity switch and stays off the network. */ +/* The failov tile owns the sockets to the other failover machine, keeps + host networking for them and holds no private key. The sign tile + signs our member certificate with the failover identity and our TLS + handshakes with the junk key, and the admin tile does the identity + switch and stays off the network. */ /* Idle socket poll spacing */ #define FD_FAILOVER_TILE_POLL_NANOS (50000L) /* 50us */ @@ -39,7 +39,7 @@ /* The promotion's replay and adopt waits also end on our clock, one second per deadline slot, so a standby whose replay froze stops. */ -#define FD_FAILOVER_DEADLINE_SLOT_NANOS (1000000000L) +#define FD_FAILOVER_DEADLINE_SLOT_NANOS ((long)FD_FAILOVER_NANOS_PER_SECOND) /* A staked contact info from another host newer than this means an active is publishing, two gossip refreshes of 7.5s. */ @@ -119,7 +119,7 @@ typedef struct fd_failover_request fd_failover_request_t; /* Our demotion. */ struct fd_failover_handoff { ulong id; /* id of our latest demotion */ - fd_failover_peer_t target; /* peer boot it is for, boot_id 0 for a demote */ + fd_failover_peer_t target; /* peer boot it is for */ ulong result; /* FD_FAILOVER_HANDOFF_* of the latest DEMOTED */ ulong delivery; /* FD_FAILOVER_DELIVERY_* */ long until; /* the demotion step's deadline on our clock, for when replay stops */ @@ -188,13 +188,17 @@ struct fd_failover_id_switch { ulong response_nonce; /* nonce of the frame being consumed */ int fresh; /* a response frame arrived and is not handled yet */ int overdue; /* the switch in flight passed its deadline, we still wait for it */ + int discard; /* set-identity ran while it was in flight, its answer is dropped */ + ulong epoch; /* failover epoch we last saw, sent with every request */ }; typedef struct fd_failover_id_switch fd_failover_id_switch_t; struct fd_failover_tile_ctx { - fd_keyguard_client_t keyguard_client[ 1 ]; /* signs the member certificate */ - int member_cert_set; /* hello carries a signed member certificate */ + fd_keyguard_client_t keyguard_client[ 1 ]; /* signs the member certificate */ + int member_cert_set; /* hello carries a signed member certificate */ + int client_only; /* no listener and no incoming handoffs */ + int target_pending; /* waiting for Admin to prepare the client's target */ fd_failover_hello_t hello; /* the HELLO we send */ ulong role; /* FD_FAILOVER_ROLE_* */ @@ -210,9 +214,9 @@ struct fd_failover_tile_ctx { long handoff_refuse_log_at; /* when we may log a refused handoff request again */ ulong peer_floor; /* highest last vote the peer reported, kept across sessions */ - ulong own_floor; /* highest vote we signed while ACTIVE this boot */ + ulong own_floor; /* highest vote this machine signed since it started */ - /* gossip_ciseen, polled unreliably for contact infos */ + /* gossip_ciseen, reliable contact info updates */ ulong gossip_in_idx; /* gossip_ciseen in link */ fd_wksp_t * gossip_in_mem; /* its workspace */ ulong gossip_in_chunk0; /* its first chunk */ @@ -236,7 +240,7 @@ struct fd_failover_tile_ctx { ulong handoff_base; /* random base for our handoff ids */ ulong handoff_cnt; /* handoff ids issued since boot */ fd_failover_handoff_t handoff; /* our demotion */ - int taken; /* the peer took the identity, only a new tenure clears this */ + int taken; /* the peer took the identity, our next tenure or a set-identity clears this */ fd_failover_tower_t peer_tower; /* stored peer tower, from a DEMOTED */ @@ -255,8 +259,9 @@ struct fd_failover_tile_ctx { ulong admin_out_chunk0; /* its first chunk */ ulong admin_out_wmark; /* its last chunk */ ulong admin_out_chunk; /* next chunk we publish to */ - fd_failover_bus_msg_t bus_req; /* the operator command copied in during_frag */ - int bus_req_fresh; /* bus_req arrived and is not handled yet */ + fd_failover_bus_msg_t bus_req; /* the operator command copied in during_frag */ + int bus_req_fresh; /* bus_req arrived and is not handled yet */ + fd_failover_operator_t bus_operator; /* copied in during_frag */ fd_failover_id_switch_t id_switch; /* the identity switch we asked the admin tile for */ @@ -275,7 +280,7 @@ struct fd_failover_tile_ctx { ulong adopt_result_id; /* the request it answers */ int adopt_result_fresh; /* adopt_result arrived and is not handled yet */ - /* tower_out, polled unreliably */ + /* tower_out, reliable vote history updates */ ulong tower_in_idx; /* tower_out in link */ fd_wksp_t * tower_in_mem; /* its workspace */ ulong tower_in_chunk0; /* its first chunk */ @@ -343,12 +348,16 @@ privileged_init( fd_topo_t const * topo, TLS. We keep only the public keys of it and of the staked [paths.identity_key]. Like every tile's identity load, the loader reads the whole file and zeroes the private half before it returns. */ - uchar const * junk_pubkey = fd_keyload_load( tile->failov.identity_key_path, 1 ); - uchar const * staked_pubkey = fd_keyload_load( tile->failov.staked_key_path, 1 ); /* public key only, like the other tiles' identity loads */ - fd_memcpy( ctx->hello.junk_pubkey, junk_pubkey, 32UL ); - fd_memcpy( ctx->hello.staked_pubkey, staked_pubkey, 32UL ); - fd_keyload_unload( junk_pubkey, 1 ); - fd_keyload_unload( staked_pubkey, 1 ); + uchar const * junk_pubkey = fd_keyload_load( tile->failov.identity_key_path, 1 ); + fd_memcpy( ctx->hello.junk_pubkey, junk_pubkey, 32UL ); + fd_keyload_unload( junk_pubkey, 1 ); + ctx->client_only = !tile->failov.staked_key_path[ 0 ]; + ctx->target_pending = ctx->client_only; + if( !ctx->client_only ) { + uchar const * staked_pubkey = fd_keyload_load( tile->failov.staked_key_path, 1 ); + fd_memcpy( ctx->hello.staked_pubkey, staked_pubkey, 32UL ); + fd_keyload_unload( staked_pubkey, 1 ); + } if( FD_UNLIKELY( fd_memeq( ctx->hello.junk_pubkey, ctx->hello.staked_pubkey, 32UL ) ) ) { FD_LOG_ERR(( "the failover junk identity `%s` is the staked [paths.identity_key], each failover machine boots under its own key", tile->failov.identity_key_path )); @@ -387,20 +396,21 @@ privileged_init( fd_topo_t const * topo, ctx->channel = fd_failover_channel_join( fd_failover_channel_new( channel_mem ) ); FD_TEST( ctx->channel ); fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 }; - if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, &ctx->hello ) ) ) { + if( FD_UNLIKELY( !ctx->target_pending && fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, &ctx->hello ) ) ) { FD_LOG_ERR(( "failover TLS setup failed for the junk identity `%s`", tile->failov.identity_key_path )); } - fd_failover_channel_init_listener( ctx->channel, tile->failov.listen_addr, tile->failov.port ); + if( !ctx->client_only ) fd_failover_channel_init_listener( ctx->channel, tile->failov.listen_addr, tile->failov.port ); ctx->session.state = fd_failover_channel_state( ctx->channel ); char junk_b58 [ FD_BASE58_ENCODED_32_SZ ]; char staked_b58[ FD_BASE58_ENCODED_32_SZ ]; fd_base58_encode_32( ctx->hello.junk_pubkey, NULL, junk_b58 ); fd_base58_encode_32( ctx->hello.staked_pubkey, NULL, staked_b58 ); - FD_LOG_NOTICE(( "failover is on, booting as a standby under junk identity `%s` for staked identity `%s`, listening on `" FD_IP4_ADDR_FMT ":%u`, " - "this machine does not vote until `failover promote` hands it the identity from the active, or `failover promote --force` takes it when no machine holds it, " - "the junk identity is this machine's own unique key, never copy it", junk_b58, staked_b58, FD_IP4_ADDR_FMT_ARGS( tile->failov.listen_addr ), (uint)tile->failov.port )); + if( ctx->client_only ) FD_LOG_NOTICE(( "failover listener is off, `failover promote ` can request an outgoing handoff while this machine retains its configured identity `%s`", junk_b58 )); + else FD_LOG_NOTICE(( "failover is on, booting as a standby under junk identity `%s` for staked identity `%s`, listening on `" FD_IP4_ADDR_FMT ":%u`, " + "this machine does not vote until `failover promote` hands it the identity from the active, or `failover promote --force` takes it when no machine holds it, " + "the junk identity is this machine's own unique key, never copy it", junk_b58, staked_b58, FD_IP4_ADDR_FMT_ARGS( tile->failov.listen_addr ), (uint)tile->failov.port )); FD_TEST( FD_SCRATCH_ALLOC_FINI( l, scratch_align() )==(ulong)scratch+scratch_footprint( tile ) ); } @@ -492,19 +502,28 @@ unprivileged_init( fd_topo_t const * topo, } } -/* Asks the sign tile to sign our member certificate with the staked key, - and installs it. */ +/* Installs the member certificate the sign tile signed, unless it was + signed with a key other than the failover identity. */ +static void +member_cert_signed( fd_failover_tile_ctx_t * ctx, + uchar const cert[ 64 ] ) { + ctx->member_cert_set = 1; + if( FD_UNLIKELY( fd_failover_channel_set_member_cert( ctx->channel, cert ) ) ) { + FD_LOG_WARNING(( "the sign tile signed our member certificate with another key, set-identity changed the failover identity, we ask again once failover restarts for it" )); + return; + } + FD_LOG_NOTICE(( "the sign tile signed our member certificate with the failover identity, this machine can pair with the other failover machine now" )); +} + +/* Asks the sign tile to sign our member certificate with the failover + identity, and installs it. */ static void request_member_cert( fd_failover_tile_ctx_t * ctx ) { uchar msg [ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ]; uchar cert[ 64 ]; fd_failover_member_cert_msg( msg, ctx->hello.junk_pubkey ); fd_keyguard_client_sign( ctx->keyguard_client, cert, msg, sizeof(msg), FD_KEYGUARD_SIGN_TYPE_ED25519 ); - if( FD_UNLIKELY( fd_failover_channel_set_member_cert( ctx->channel, cert ) ) ) { - FD_LOG_ERR(( "the sign tile signed our member certificate with a key other than the staked [paths.identity_key]" )); - } - ctx->member_cert_set = 1; - FD_LOG_NOTICE(( "the sign tile signed our member certificate with the staked identity, this machine can pair with the other failover machine now" )); + member_cert_signed( ctx, cert ); } /* Updates our session state when the channel's session changes. */ @@ -610,13 +629,6 @@ promote_expired( fd_failover_tile_ctx_t const * ctx, return 1; } -/* What the operator knows our demotion as, a handoff when a peer asked - for it, else a demotion. */ -static inline char const * -demotion_kind( fd_failover_tile_ctx_t const * ctx ) { - return ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE ? "handoff" : "demotion"; -} - /* Marks the identity switch in flight as overdue, flags stuck and warns once. We keep waiting for the admin tile's answer. */ static void @@ -625,7 +637,7 @@ switch_overdue( fd_failover_tile_ctx_t * ctx ) { if( FD_LIKELY( ctx->id_switch.overdue ) ) return; ctx->id_switch.overdue = 1; char demotion[ 48 ]; - fd_cstr_printf( demotion, sizeof(demotion), NULL, "%s %lu", demotion_kind( ctx ), ctx->handoff.id ); + fd_cstr_printf( demotion, sizeof(demotion), NULL, "handoff %lu", ctx->handoff.id ); FD_LOG_WARNING(( "%s: identity switch %lu to %s is overdue, its outcome is unknown, we keep waiting and refuse another switch, check the admin and sign tile logs", ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? ctx->promotion.label : demotion, ctx->id_switch.request_id, ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? "staked" : "junk" )); @@ -736,7 +748,7 @@ control_sent( fd_failover_tile_ctx_t * ctx, case (ushort)FD_FAILOVER_MSG_DEMOTED: if( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_PENDING ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_SENT; if( now>=ctx->handoff.log_at ) { - ctx->handoff.log_at = fd_long_sat_add( now, 60000000000L ); + ctx->handoff.log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_NOTICE(( "handoff %lu: sent final state (DEMOTED) to peer boot %016lx, last vote slot %lu, we remain standby waiting for its answer", ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot )); } @@ -744,7 +756,7 @@ control_sent( fd_failover_tile_ctx_t * ctx, case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK: case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: if( now>=ctx->reply.log_at ) { - ctx->reply.log_at = fd_long_sat_add( now, 60000000000L ); + ctx->reply.log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_NOTICE(( "handoff %lu: sent %s to peer boot %016lx, %s, waiting for its final confirmation", ctx->reply.handoff_id, ctx->tx.type==FD_FAILOVER_MSG_PROMOTE_ACK ? "ACK" : "refusal", ctx->reply.peer.boot_id, ctx->role==FD_FAILOVER_ROLE_ACTIVE ? "we hold the staked identity" : "we remain standby" )); @@ -754,7 +766,7 @@ control_sent( fd_failover_tile_ctx_t * ctx, fd_failover_handoff_result_t result; fd_memcpy( &result, ctx->tx.payload, sizeof(result) ); if( now>=ctx->handoff.result_log_at ) { - ctx->handoff.result_log_at = fd_long_sat_add( now, 60000000000L ); + ctx->handoff.result_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); char const * hint; FD_LOG_NOTICE(( "handoff %lu: sent %s confirmation%s%s%s to peer boot %016lx, closing connection after queued bytes drain", result.handoff_id, result.result==FD_ADMINCTL_RESULT_SUCCESS ? "success" : "refusal", @@ -781,20 +793,30 @@ pending_flush( fd_failover_tile_ctx_t * ctx, ctx->tx.valid = 0; } -/* Keep the tower of our latest vote from the tower's vote transaction. +/* Keep the tower of our latest vote for the failover identity. It is the final tower we hand over at a demotion. A malformed transaction is logged and skipped, the previous tower stays. */ static void prepare_consensus( fd_failover_tile_ctx_t * ctx, fd_tower_slot_done_t const * done ) { - if( FD_UNLIKELY( !done->vote_txn_sz || done->vote_txn_sz>sizeof(done->vote_txn) ) ) { - FD_LOG_WARNING(( "tower produced an invalid vote transaction size" )); - return; - } + if( FD_UNLIKELY( ctx->target_pending ) ) return; + + uchar txn_mem[ FD_TXN_MAX_SZ ] __attribute__((aligned(alignof(fd_txn_t)))); + int parsed = done->vote_txn_sz && done->vote_txn_sz<=sizeof(done->vote_txn) && + fd_txn_parse( done->vote_txn, done->vote_txn_sz, txn_mem, NULL ); + /* The fee payer is the node identity, even with a separate vote authority. + A client can vote under its boot key until it adopts the target history. + Check the transaction identity because queued votes can outlive a switch. */ + if( FD_LIKELY( parsed ) ) { + fd_acct_addr_t const * accounts = fd_txn_get_acct_addrs( (fd_txn_t const *)txn_mem, done->vote_txn ); + if( FD_UNLIKELY( !fd_memeq( accounts[ 0 ].b, ctx->hello.staked_pubkey, 32UL ) ) ) return; + } + /* An unreadable transaction still raises the floor conservatively. */ + ctx->last_vote_slot = done->vote_slot; + if( FD_LIKELY( ctx->own_floor==FD_FAILOVER_SLOT_NULL || done->vote_slot>ctx->own_floor ) ) ctx->own_floor = done->vote_slot; - uchar txn_mem[ FD_TXN_MAX_SZ ] __attribute__((aligned(alignof(fd_txn_t)))); fd_compact_tower_sync_serde_t serde; - if( FD_UNLIKELY( !fd_txn_parse( done->vote_txn, done->vote_txn_sz, txn_mem, NULL ) || + if( FD_UNLIKELY( !parsed || !fd_txn_parse_simple_vote( (fd_txn_t const *)txn_mem, done->vote_txn, &serde ) ) ) { FD_LOG_WARNING(( "tower produced an invalid vote transaction" )); return; @@ -833,14 +855,7 @@ consume_slot_done( fd_failover_tile_ctx_t * ctx, ctx->replay_slot = done->replay_slot; if( FD_LIKELY( done->root_slot!=FD_FAILOVER_SLOT_NULL ) ) ctx->root_slot = done->root_slot; if( FD_LIKELY( done->has_vote_txn && done->vote_slot!=FD_FAILOVER_SLOT_NULL ) ) { - ctx->last_vote_slot = done->vote_slot; - /* Admin can unhalt the new signer before its switch response reaches - this tile. Those first votes already belong to the new tenure. */ - if( FD_LIKELY( ctx->role==FD_FAILOVER_ROLE_ACTIVE || ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH ) ) { - /* Any later promotion has to cover the votes we signed. */ - if( FD_LIKELY( ctx->own_floor==FD_FAILOVER_SLOT_NULL || done->vote_slot>ctx->own_floor ) ) ctx->own_floor = done->vote_slot; - prepare_consensus( ctx, done ); - } + prepare_consensus( ctx, done ); } } @@ -879,6 +894,7 @@ request_switch( fd_failover_tile_ctx_t * ctx, out->nonce = ctx->id_switch.request_id; fd_failover_switch_req_t req; fd_memcpy( req.identity, key==FD_FAILOVER_SWITCH_KEY_STAKED ? ctx->hello.staked_pubkey : ctx->hello.junk_pubkey, 32UL ); + req.epoch = ctx->id_switch.epoch; fd_memcpy( out->payload, &req, sizeof(req) ); ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); fd_stem_publish( stem, ctx->admin_out_idx, FD_FAILOVER_BUS_SWITCH_REQ, ctx->admin_out_chunk, sizeof(*out), 0UL, tspub, tspub ); @@ -889,9 +905,9 @@ request_switch( fd_failover_tile_ctx_t * ctx, return ctx->id_switch.request_id; } -/* Records the result of the switch we asked for. A reply for some other - request is dropped, otherwise an old reply could be mistaken for a - finished demotion. */ +/* Ends the switch in flight when nonce answers it and returns 1 if the + caller should take the result. We drop a reply to another request, + and the reply to a switch that set-identity overtook. */ static int switch_answer( fd_failover_tile_ctx_t * ctx, ulong nonce ) { @@ -902,10 +918,78 @@ switch_answer( fd_failover_tile_ctx_t * ctx, } /* A matching response ends the pending switch. */ ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT; - ctx->id_switch.fresh = 1; + if( FD_UNLIKELY( ctx->id_switch.discard ) ) { + ctx->id_switch.discard = 0; + return 0; + } + ctx->id_switch.fresh = 1; return 1; } +/* set-identity installed a key while failover is on, so we start over + as if this machine had rebooted with that key. We take a new boot id, + so the other machine treats anything bound to the old one as from a + restarted peer, drop everything in flight, and take the failover + identity and role that set-identity left. A peer with another + failover identity fails HELLO. */ +static void +operator_switched( fd_failover_tile_ctx_t * ctx, + fd_failover_operator_t const * operator ) { + if( FD_UNLIKELY( operator->epoch<=ctx->id_switch.epoch ) ) return; + ctx->id_switch.epoch = operator->epoch; + ctx->id_switch.discard = ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT; + ctx->id_switch.fresh = 0; + ctx->id_switch.overdue = 0; + + long now = fd_failover_clock(); + fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); + fd_failover_channel_hangup( ctx->channel, now ); + + fd_memset( &ctx->request, 0, sizeof(ctx->request) ); + fd_memset( &ctx->handoff, 0, sizeof(ctx->handoff) ); + fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) ); + fd_memset( &ctx->reply, 0, sizeof(ctx->reply) ); + fd_memset( &ctx->tx, 0, sizeof(ctx->tx) ); + ctx->peer_tower.valid = 0; + ctx->current_tower.valid = 0; + ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; + ctx->tower_gap = 0; + ctx->taken = 0; + ctx->last_requested = 0; + ctx->cmd_addr = 0U; + ctx->cmd_port = (ushort)0; + ctx->session.close_after_send = 0; + ctx->adopt_result_fresh = 0; + ctx->action = FD_FAILOVER_ACTION_IDLE; + ctx->stuck = 0; + ctx->deadline_slot = FD_FAILOVER_SLOT_NULL; + /* What gossip told us is about the old failover identity. */ + if( FD_UNLIKELY( !fd_memeq( operator->failover_identity, ctx->hello.staked_pubkey, 32UL ) ) ) { + ctx->staked_addr = 0U; + ctx->staked_seen_at = 0L; + } + + ulong boot_id = ctx->hello.boot_id; + do FD_TEST( fd_rng_secure( &ctx->hello.boot_id, sizeof(ulong) ) ); while( FD_UNLIKELY( !ctx->hello.boot_id || ctx->hello.boot_id==boot_id ) ); + fd_memcpy( ctx->hello.staked_pubkey, operator->failover_identity, 32UL ); + int staked = !fd_memeq( operator->identity, ctx->hello.junk_pubkey, 32UL ); + ctx->role = staked ? FD_FAILOVER_ROLE_ACTIVE : FD_FAILOVER_ROLE_STANDBY; + ctx->hello.role = (uchar)ctx->role; + fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 }; + if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, &ctx->hello ) ) ) { + FD_LOG_ERR(( "failover TLS setup failed after set-identity" )); + } + ctx->member_cert_set = 0; + ctx->target_pending = 0; + sync_session( ctx ); + + FD_BASE58_ENCODE_32_BYTES( operator->identity, identity ); + FD_BASE58_ENCODE_32_BYTES( operator->failover_identity, failover_identity ); + FD_LOG_WARNING(( "Admin selected failover identity `%s` with installed identity `%s`, this machine is a %s with a new boot %016lx, " + "anything in progress was dropped and the other machine sees a restart", + failover_identity, identity, staked ? "active" : "standby", ctx->hello.boot_id )); +} + /* Our DEMOTED got its response or cannot get one any more. */ static void handoff_resolved( fd_failover_tile_ctx_t * ctx, @@ -938,24 +1022,20 @@ demotion_deadline_start( fd_failover_tile_ctx_t * ctx ) { ctx->handoff.until = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_CHANNEL_IDLE_NANOS ); } -/* Give up the staked identity. The switch itself is asked for from - step_controller, after the command's response went out. A handoff - sends DEMOTED to the peer boot we see now, a demote sends nothing. */ +/* Give up the staked identity for the peer that asked, on the session + it asked on. The switch itself is asked for from step_controller, + DEMOTED goes to the peer boot we see now. */ static void start_demotion( fd_failover_tile_ctx_t * ctx, - int handoff ) { - int send = handoff && fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED; + ulong handoff_id ) { fd_memset( &ctx->handoff, 0, sizeof(ctx->handoff) ); - ctx->handoff.id = ctx->handoff_base+(++ctx->handoff_cnt); - ctx->handoff.result = FD_FAILOVER_HANDOFF_NONE; /* until its DEMOTED goes out */ - ctx->handoff.delivery = send ? FD_FAILOVER_DELIVERY_PENDING : FD_FAILOVER_DELIVERY_NONE; - if( send ) { - ctx->handoff.target.boot_id = ctx->session.peer_boot_id; - fd_memcpy( ctx->handoff.target.junk, fd_failover_channel_peer_hello( ctx->channel )->junk_pubkey, 32UL ); - } + ctx->handoff.id = handoff_id; + ctx->handoff.result = FD_FAILOVER_HANDOFF_NONE; /* until its DEMOTED goes out */ + ctx->handoff.delivery = FD_FAILOVER_DELIVERY_PENDING; + ctx->handoff.target.boot_id = ctx->session.peer_boot_id; + fd_memcpy( ctx->handoff.target.junk, fd_failover_channel_peer_hello( ctx->channel )->junk_pubkey, 32UL ); ctx->last_requested = 0; ctx->stuck = 0; - if( FD_UNLIKELY( !send ) ) FD_LOG_NOTICE(( "demotion %lu started without a handoff, giving up the staked identity, no DEMOTED will be sent", ctx->handoff.id )); demotion_deadline_start( ctx ); /* Start the junk key switch before sending any final state. */ ctx->action = FD_FAILOVER_ACTION_DEMOTE_SWITCH; @@ -971,33 +1051,15 @@ final_tower_ok( fd_failover_tile_ctx_t const * ctx ) { } /* Called once the junk key is installed everywhere. Our latest tower is - the final one. A bound handoff hands it to the peer. A standalone - demote requires shared-source recovery. */ + the final one, we hand it to the peer. */ static void demotion_switched( fd_failover_tile_ctx_t * ctx ) { /* The junk key is installed and the Tower has drained. */ set_role( ctx, FD_FAILOVER_ROLE_STANDBY ); - FD_LOG_NOTICE(( "%s %lu: the tower drained, we are a standby now", demotion_kind( ctx ), ctx->handoff.id )); + FD_LOG_NOTICE(( "handoff %lu: the tower drained, we are a standby now", ctx->handoff.id )); int final_ok = final_tower_ok( ctx ); - if( FD_UNLIKELY( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_NONE ) ) { - /* Either member may recover after a standalone demote. There is no - bound exchange to tell us whether the peer has since voted. */ - ctx->peer_tower.valid = 0; - if( FD_UNLIKELY( !final_ok ) ) { - /* Without final state, recovery needs the vote account. */ - FD_LOG_WARNING(( "demotion %lu has no final tower for its last vote %lu, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->last_vote_slot )); - /* End demotion without sending final state. */ - demotion_abort( ctx ); - return; - } - FD_LOG_NOTICE(( "demotion %lu is done, neither machine holds the staked identity now, nothing votes until `failover promote --force` runs on one of them", ctx->handoff.id )); - /* A standalone demotion finishes here. */ - ctx->action = FD_FAILOVER_ACTION_IDLE; - return; - } - ctx->handoff.payload_sz = final_ok ? fd_failover_demoted_encode( ctx->handoff.payload, ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot, ctx->current_tower.state, ctx->current_tower.sz ) @@ -1066,7 +1128,7 @@ control_refusal( ulong result, char const ** hint ) { switch( result ) { case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: - *hint = "`promote` runs on a standby, `demote` on the active, see `failover status`"; + *hint = "`failover promote` runs on a standby, see `failover status`"; return "BAD_ROLE"; case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: *hint = "a transition or key switch is running, wait for it, `failover status` shows it"; @@ -1259,7 +1321,7 @@ step_demote_drain( fd_failover_tile_ctx_t * ctx ) { signing. */ if( FD_UNLIKELY( !fd_seq_ge( fd_seq_inc( ctx->tower_seen_seq, 1UL ), ctx->id_switch.result.tower_watermark ) ) ) { if( FD_UNLIKELY( expired ) ) { - FD_LOG_WARNING(( "%s %lu: the tower did not report its last votes in time, up to seq %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", demotion_kind( ctx ), ctx->handoff.id, ctx->id_switch.result.tower_watermark )); + FD_LOG_WARNING(( "handoff %lu: the tower did not report its last votes in time, up to seq %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->id_switch.result.tower_watermark )); ctx->id_switch.fresh = 0; ctx->id_switch.overdue = 0; set_role( ctx, FD_FAILOVER_ROLE_STANDBY ); @@ -1286,7 +1348,7 @@ step_demote_switch( fd_failover_tile_ctx_t * ctx, demotion_abort( ctx ); return; } - FD_LOG_NOTICE(( "%s %lu: asked the admin tile to install the junk identity", demotion_kind( ctx ), ctx->handoff.id )); + FD_LOG_NOTICE(( "handoff %lu: asked the admin tile to install the junk identity", ctx->handoff.id )); return; } if( FD_LIKELY( !ctx->id_switch.fresh ) ) { @@ -1299,14 +1361,14 @@ step_demote_switch( fd_failover_tile_ctx_t * ctx, final state. */ ctx->id_switch.fresh = 0; ctx->id_switch.overdue = 0; - FD_LOG_WARNING(( "the identity switch for %s %lu failed (%s), we keep the staked identity and stay the active", demotion_kind( ctx ), ctx->handoff.id, + FD_LOG_WARNING(( "the identity switch for handoff %lu failed (%s), we keep the staked identity and stay the active", ctx->handoff.id, ctx->id_switch.result.result==FD_FAILOVER_SWITCH_ERR_DISABLED ? "failover disabled" : "unknown" )); /* End demotion without sending final state. */ demotion_abort( ctx ); return; } /* The junk key is installed everywhere. */ - FD_LOG_NOTICE(( "%s %lu: the junk identity is installed, waiting for the tower to drain up to seq %lu", demotion_kind( ctx ), ctx->handoff.id, ctx->id_switch.result.tower_watermark )); + FD_LOG_NOTICE(( "handoff %lu: the junk identity is installed, waiting for the tower to drain up to seq %lu", ctx->handoff.id, ctx->id_switch.result.tower_watermark )); ctx->action = FD_FAILOVER_ACTION_DEMOTE_DRAIN; demotion_deadline_start( ctx ); step_demote_drain( ctx ); @@ -1602,7 +1664,7 @@ request_next( fd_failover_tile_ctx_t const * ctx ) { "and if the active is down and cannot sign `failover promote --force` takes over"; return "the active may already have given up the identity, check `failover status` there, `failover promote` here resumes the request"; case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: - if( ctx->role==FD_FAILOVER_ROLE_ACTIVE ) return "this machine holds the staked identity and votes, `failover promote --yes` here asks the old active again to confirm"; + if( ctx->role==FD_FAILOVER_ROLE_ACTIVE ) return "this machine holds the staked identity and votes, `failover promote` here asks the old active again to confirm"; if( ctx->reply.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) return "this machine refused the handoff because another machine may hold the staked identity, " "check `failover status` on both machines, `failover promote` here asks the old active again to confirm"; return "neither machine holds the staked identity, `failover promote` here asks the old active again to confirm, " @@ -1659,7 +1721,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, fd_failover_handoff_request_t request; if( FD_UNLIKELY( !fd_failover_handoff_request_decode( &request, ctx->rx, payload_sz ) ) ) break; fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); - if( FD_UNLIKELY( request.target_boot_id!=ctx->hello.boot_id ) ) { + if( FD_UNLIKELY( ctx->client_only || request.target_boot_id!=ctx->hello.boot_id ) ) { handoff_result( ctx, request.handoff_id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); return; } @@ -1685,7 +1747,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, /* Decline the request without changing the current role or action. */ if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { if( result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER && now>=ctx->handoff_refuse_log_at ) { - ctx->handoff_refuse_log_at = fd_long_sat_add( now, 60000000000L ); + ctx->handoff_refuse_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); char tip[ 32 ], last[ 32 ]; char const * why = ctx->tower_gap ? "a vote-history update was skipped, the final state cannot be verified" : ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ? "no vote is known to the failover controller, it cannot determine whether this machine has not voted or is unable to vote" : @@ -1695,7 +1757,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, request.handoff_id, why, slot_text( ctx->last_vote_slot, last ), slot_text( ctx->current_tower.valid ? ctx->current_tower.tip : FD_FAILOVER_SLOT_NULL, tip ) )); } else if( result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND && now>=ctx->handoff_refuse_log_at ) { - ctx->handoff_refuse_log_at = fd_long_sat_add( now, 60000000000L ); + ctx->handoff_refuse_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); char replay[ 32 ]; FD_LOG_WARNING(( "handoff %lu refused: the standby's replay is at slot %s, more than %lu slots behind our last vote at slot %lu, this machine keeps the staked identity, " "run `failover promote` there again once it has caught up", @@ -1705,8 +1767,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, return; } /* An accepted request starts demotion on the active. */ - start_demotion( ctx, 1 ); - ctx->handoff.id = request.handoff_id; + start_demotion( ctx, request.handoff_id ); char member[ FD_BASE58_ENCODED_32_SZ ]; fd_base58_encode_32( peer->junk_pubkey, NULL, member ); FD_LOG_NOTICE(( "handoff %lu: accepted request from member %s boot %016lx, preparing to give up the staked identity", ctx->handoff.id, member, ctx->handoff.target.boot_id )); @@ -1937,7 +1998,7 @@ peer_poll( fd_failover_tile_ctx_t * ctx, ctx->request.sent = 1; ctx->request.send_cnt = fd_ulong_sat_add( ctx->request.send_cnt, 1UL ); if( now>=ctx->request.log_at ) { - ctx->request.log_at = fd_long_sat_add( now, 60000000000L ); + ctx->request.log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); char member[ FD_BASE58_ENCODED_32_SZ ]; fd_base58_encode_32( ctx->request.peer.junk, NULL, member ); FD_LOG_NOTICE(( "handoff %lu: sent REQUEST to authenticated member %s boot %016lx, %s (send %lu)", @@ -2000,24 +2061,9 @@ static ulong apply_control( fd_failover_tile_ctx_t * ctx, fd_adminctl_failover_req_t const * req, long now ) { + if( FD_UNLIKELY( ctx->target_pending && req->cmd!=FD_ADMINCTL_FAILOVER_CMD_STATUS ) ) return FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED; switch( req->cmd ) { - case FD_ADMINCTL_FAILOVER_CMD_DEMOTE: { - if( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->request.id && - ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ) { - FD_LOG_NOTICE(( "handoff %lu: operator demote ends the wait for confirmation, giving up the staked identity locally", ctx->request.id )); - /* Operator demotion ends the confirmation wait before starting a local switch. */ - request_end( ctx, now, FD_FAILOVER_HANDOFF_DECLINED ); - } - /* Drop the identity without handing it over. The peer is not asked - or told anything, the other machine then needs fenced recovery. */ - if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE ) ) return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; - if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_ACTIVE ) ) return FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; - /* An idle active starts a standalone demotion. */ - start_demotion( ctx, 0 ); - return FD_ADMINCTL_RESULT_SUCCESS; - } - case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: { int paused = request_paused( ctx ) && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT; if( FD_UNLIKELY( paused && ctx->request.peer_restarted ) ) { @@ -2116,13 +2162,13 @@ apply_control( fd_failover_tile_ctx_t * ctx, return FD_ADMINCTL_RESULT_UNSUPPORTED; } -/* The failover status response. */ +/* The `failover status` response. */ static void status_snapshot( fd_failover_tile_ctx_t const * ctx, long now, fd_adminctl_failover_status_resp_t * resp ) { fd_adminctl_failover_status_resp_init( resp ); - resp->enabled = 1; + resp->enabled = ctx->client_only ? 2 : 1; resp->role = (uchar)ctx->role; resp->action = (uchar)ctx->action; resp->stuck = (uchar)!!ctx->stuck; @@ -2139,7 +2185,7 @@ status_snapshot( fd_failover_tile_ctx_t const * ctx, resp->handoff_result = (uchar)( ctx->last_requested ? ctx->request.result : ctx->handoff.result ); resp->promote_source = (uchar)promote_source( ctx ); resp->promote_floor = coverage_floor( ctx ); - resp->promote_result = promote_guard( ctx, now, 0 ); + resp->promote_result = ctx->target_pending ? FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED : promote_guard( ctx, now, 0 ); resp->mode = ctx->hello.mode; resp->request_paused = (uchar)request_paused( ctx ); } @@ -2163,20 +2209,18 @@ serve_bus_request( fd_failover_tile_ctx_t * ctx, fd_memcpy( out->payload, &status, sizeof(status) ); } else { char const * cmd_name = fd_adminctl_failover_cmd_typed( req.cmd ); - FD_LOG_NOTICE(( "`failover %s%s%s` received", cmd_name, + FD_LOG_NOTICE(( "`%s%s%s` received", cmd_name, ( req.flags & FD_ADMINCTL_FAILOVER_FLAG_YES ) ? " --yes" : "", ( req.flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ) ? " --force" : "" )); ulong result = apply_control( ctx, &req, now ); if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { char const * hint; char const * name = control_refusal( result, &hint ); - FD_LOG_WARNING(( "`failover %s` refused with %s, %s", cmd_name, name, hint )); + FD_LOG_WARNING(( "`%s` refused with %s, %s", cmd_name, name, hint )); } - /* An accepted promote or demote says which handoff to follow. */ ulong handoff_id = 0UL; if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) { - if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) handoff_id = ctx->request.id; - else if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_DEMOTE ) handoff_id = ctx->handoff.id; + if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) handoff_id = ctx->request.id; } fd_adminctl_failover_control_resp_t response = { .version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, @@ -2230,10 +2274,8 @@ before_frag( fd_failover_tile_ctx_t * ctx, } if( FD_LIKELY( in_idx==ctx->tower_in_idx ) ) { /* A skipped frag may have been a vote. A slot done counts toward the - watermark in after_frag, once the stem has checked the copy. One - abandoned to an overrun must not count, or the final tower would be - taken from the cache with that vote missing. Other frags hold no - vote and count here. */ + watermark in after_frag, once its vote is in the cache. Other + frags hold no vote and count here. */ if( FD_UNLIKELY( ctx->tower_seen_seq!=ULONG_MAX && seq!=fd_seq_inc( ctx->tower_seen_seq, 1UL ) ) ) ctx->tower_gap = 1; if( FD_UNLIKELY( sig!=FD_TOWER_SIG_SLOT_DONE ) ) { ctx->tower_seen_seq = seq; @@ -2242,9 +2284,9 @@ before_frag( fd_failover_tile_ctx_t * ctx, return 0; } if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { - /* Commands and switch responses. Dropping a switch response here would - leave the switch hanging forever. */ - return sig!=FD_FAILOVER_BUS_REQUEST && sig!=FD_FAILOVER_BUS_SWITCH_RESP; + /* Commands, switch responses and set-identity. Dropping a switch + response here would leave the switch hanging forever. */ + return sig!=FD_FAILOVER_BUS_REQUEST && sig!=FD_FAILOVER_BUS_SWITCH_RESP && sig!=FD_FAILOVER_BUS_OPERATOR; } return 0; } @@ -2274,6 +2316,10 @@ during_frag( fd_failover_tile_ctx_t * ctx, ctx->id_switch.response_nonce = msg->nonce; return; } + if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_OPERATOR ) ) { + fd_memcpy( &ctx->bus_operator, msg->payload, sizeof(ctx->bus_operator) ); + return; + } fd_memcpy( &ctx->bus_req, msg, sizeof(fd_failover_bus_msg_t) ); return; } @@ -2288,8 +2334,7 @@ during_frag( fd_failover_tile_ctx_t * ctx, if( FD_UNLIKELY( chunkgossip_in_chunk0 || chunk>ctx->gossip_in_wmark || sz>ctx->gossip_in_mtu ) ) { FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->gossip_in_chunk0, ctx->gossip_in_wmark )); } - /* The link is unreliable, so we only copy here and after_frag commits - once the stem has checked the copy. A remove leaves the socket + /* Copy here and commit in after_frag. A remove leaves the socket zero. */ fd_gossip_update_message_t const * msg = fd_chunk_to_laddr_const( ctx->gossip_in_mem, chunk ); fd_memcpy( ctx->gossip_origin, msg->origin, 32UL ); @@ -2321,6 +2366,11 @@ after_frag( fd_failover_tile_ctx_t * ctx, /* The drain still reads the last switch result, so we copy a response into it only after the stem and nonce checks pass. */ if( FD_LIKELY( switch_answer( ctx, ctx->id_switch.response_nonce ) ) ) ctx->id_switch.result = ctx->id_switch.response; + if( FD_UNLIKELY( ctx->id_switch.response.result==FD_FAILOVER_SWITCH_ERR_STALE ) ) operator_switched( ctx, &ctx->id_switch.response.operator ); + return; + } + if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_OPERATOR ) ) { + operator_switched( ctx, &ctx->bus_operator ); return; } /* Send the response from after_credit, where a publish credit is available. */ @@ -2341,7 +2391,7 @@ after_credit( fd_failover_tile_ctx_t * ctx, fd_stem_context_t * stem, int * opt_poll_in FD_PARAM_UNUSED, int * charge_busy ) { - if( FD_UNLIKELY( !ctx->member_cert_set ) ) request_member_cert( ctx ); + if( FD_UNLIKELY( !ctx->target_pending && !ctx->member_cert_set ) ) request_member_cert( ctx ); if( FD_UNLIKELY( ctx->slot_done_fresh ) ) { ctx->slot_done_fresh = 0; consume_slot_done( ctx, &ctx->slot_done ); @@ -2349,7 +2399,7 @@ after_credit( fd_failover_tile_ctx_t * ctx, step_controller( ctx, stem ); long now = fd_failover_clock(); - if( FD_LIKELY( now>=ctx->poll_at ) ) { + if( FD_LIKELY( !ctx->target_pending && now>=ctx->poll_at ) ) { int busy = 0; peer_poll( ctx, now, &busy ); ctx->poll_at = busy ? now : now+FD_FAILOVER_TILE_POLL_NANOS; diff --git a/src/discof/failover/test_failover_tile.c b/src/discof/failover/test_failover_tile.c index f3787fad524..8bd61a9d482 100644 --- a/src/discof/failover/test_failover_tile.c +++ b/src/discof/failover/test_failover_tile.c @@ -1,5 +1,6 @@ #include "fd_failover_channel.c" #include "fd_failover_tile.c" +#include "../../ballet/ed25519/fd_ed25519.h" #include @@ -227,6 +228,173 @@ test_demote_drain( void ) { controller_fini( ctx ); } +/* Deliver an OPERATOR frame from the admin tile. */ +static uchar admin_in_mem[ 1024 ] __attribute__((aligned(128))); + +static void +operator_frame( fd_failover_tile_ctx_t * ctx, + ulong epoch, + uchar key, + uchar failover_key ) { + ctx->admin_in_idx = 1UL; + ctx->admin_in_mem = (fd_wksp_t *)admin_in_mem; + ctx->admin_in_chunk0 = 0UL; + ctx->admin_in_wmark = 0UL; + fd_failover_bus_msg_t * msg = (fd_failover_bus_msg_t *)admin_in_mem; + fd_memset( msg, 0, sizeof(*msg) ); + fd_failover_operator_t operator = { .epoch=epoch }; + fd_memset( operator.identity, key, 32UL ); + fd_memset( operator.failover_identity, failover_key, 32UL ); + fd_memcpy( msg->payload, &operator, sizeof(operator) ); + FD_TEST( !before_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR ) ); + during_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR, 0UL, sizeof(*msg), 0UL ); + after_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR, sizeof(*msg), 0UL, 0UL, stem ); +} + +/* test_operator_switch: set-identity during a demotion. We restart on + the new key with a new boot and nothing in flight, the answer to the + switch we asked for before is dropped, the next request has the new + failover epoch, and a request bound to our old boot cannot demote us. */ +static void +test_operator_switch( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + make_tower( &ctx->current_tower, 99UL ); + ctx->own_floor = 99UL; + fd_failover_handoff_request_t req = { .handoff_id=47UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + step_controller( ctx, stem ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK ); + ulong nonce = ctx->id_switch.request_id; + + /* set-identity installs the junk key. */ + operator_frame( ctx, 1UL, 0x11, 0x5A ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && ctx->hello.role==FD_FAILOVER_ROLE_STANDBY ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->stuck && !ctx->tx.valid && !ctx->handoff.id && !ctx->request.id ); + FD_TEST( ctx->hello.boot_id!=OUR_BOOT_ID && ctx->channel->self_hello.boot_id==ctx->hello.boot_id ); + FD_TEST( !ctx->member_cert_set && !ctx->current_tower.valid && ctx->own_floor==99UL ); + FD_TEST( ctx->id_switch.epoch==1UL && ctx->id_switch.discard && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK ); + FD_TEST( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ); + + /* The late answer ends the old switch and is dropped. */ + ctx->id_switch.result.result = FD_FAILOVER_SWITCH_OK; + FD_TEST( !switch_answer( ctx, nonce ) ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.fresh && !ctx->id_switch.discard ); + + /* The same or an older frame changes nothing. */ + ulong boot_id = ctx->hello.boot_id; + operator_frame( ctx, 1UL, 0x5A, 0x5A ); + FD_TEST( ctx->hello.boot_id==boot_id && ctx->role==FD_FAILOVER_ROLE_STANDBY ); + + /* The next switch request has the new failover epoch. */ + FD_TEST( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_STAKED )!=ULONG_MAX ); + fd_failover_switch_req_t sent; + fd_memcpy( &sent, ((fd_failover_bus_msg_t const *)bus_mem)->payload, sizeof(sent) ); + FD_TEST( sent.epoch==1UL ); + + /* The request crossed another set-identity, with the staked key, whose + OPERATOR frame has not arrived, and the STALE refusal catches us up. */ + ctx->admin_in_idx = 1UL; + ctx->id_switch.response.result = FD_FAILOVER_SWITCH_ERR_STALE; + ctx->id_switch.response.operator.epoch = 2UL; + fd_memset( ctx->id_switch.response.operator.identity, 0x5A, 32UL ); + fd_memset( ctx->id_switch.response.operator.failover_identity, 0x5A, 32UL ); + ctx->id_switch.response_nonce = ctx->id_switch.request_id; + after_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_SWITCH_RESP, sizeof(fd_failover_bus_msg_t), 0UL, 0UL, stem ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->id_switch.epoch==2UL && ctx->hello.boot_id!=boot_id ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.fresh && ctx->action==FD_FAILOVER_ACTION_IDLE ); + + /* A request bound to our first boot is refused, one to this boot is + served once we have a final tower. */ + pair( ctx, 78UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE ); + ctx->tx.valid = 0; + ctx->session.close_after_send = 0; + ctx->last_vote_slot = 120UL; + make_tower( &ctx->current_tower, 120UL ); + fd_failover_handoff_request_t fresh = { .handoff_id=48UL, .target_boot_id=ctx->hello.boot_id, .replay_slot=120UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &fresh, sizeof(fresh) ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->handoff.id==48UL ); + controller_fini( ctx ); +} + +/* test_operator_failover_identity: HELLO has the failover identity + OPERATOR gives, the role follows the installed key, and a member cert + signed with another key waits for the next OPERATOR. */ +static void +test_operator_failover_identity( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); + ctx->staked_seen_at = 900L; + operator_frame( ctx, 1UL, 0x77, 0x77 ); + uchar third[ 32 ]; fd_memset( third, 0x77, 32UL ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->hello.role==FD_FAILOVER_ROLE_ACTIVE ); + FD_TEST( fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) && fd_memeq( ctx->channel->self_hello.staked_pubkey, third, 32UL ) ); + FD_TEST( !ctx->staked_addr && !ctx->staked_seen_at && !ctx->member_cert_set ); + + /* A junk set-identity keeps the failover identity, the configured key does not come back. */ + operator_frame( ctx, 2UL, 0x11, 0x77 ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) ); + + /* Any key but junk makes us the active, also one that is not the + failover identity in the frame. */ + operator_frame( ctx, 3UL, 0x5A, 0x77 ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) ); + + /* A cert over our junk key signed with the configured key, as the sign + tile would sign it before this OPERATOR, is refused by the channel and + we wait for the next OPERATOR. */ + uchar secret[ 32 ]; fd_memset( secret, 0x5A, 32UL ); + uchar public[ 32 ]; + fd_sha512_t sha[ 1 ]; + FD_TEST( fd_sha512_join( fd_sha512_new( sha ) ) ); + fd_ed25519_public_from_private( public, secret, sha ); + uchar msg[ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ], cert[ 64 ]; + fd_failover_member_cert_msg( msg, ctx->hello.junk_pubkey ); + fd_ed25519_sign( cert, msg, sizeof(msg), public, secret, sha ); + member_cert_signed( ctx, cert ); + FD_TEST( ctx->member_cert_set && !ctx->channel->member_cert_set ); + + /* With a real key other than junk and staked in the next OPERATOR, a + cert it signed is ours, and HELLO passes against a peer with the + same key and fails against one still on the configured key. */ + uchar third_secret[ 32 ]; fd_memset( third_secret, 0x66, 32UL ); + uchar third_public[ 32 ]; + fd_ed25519_public_from_private( third_public, third_secret, sha ); + operator_frame( ctx, 4UL, 0x11, 0x77 ); + fd_memcpy( ctx->hello.staked_pubkey, third_public, 32UL ); + fd_memcpy( ctx->channel->self_hello.staked_pubkey, third_public, 32UL ); + fd_ed25519_sign( cert, msg, sizeof(msg), third_public, third_secret, sha ); + member_cert_signed( ctx, cert ); + FD_TEST( ctx->channel->member_cert_set ); + fd_failover_hello_t peer = ctx->channel->self_hello; + peer.role = FD_FAILOVER_ROLE_ACTIVE; + peer.boot_id = 77UL; + fd_memset( peer.junk_pubkey, 0x22, 32UL ); + FD_TEST( fd_failover_hello_check( &ctx->channel->self_hello, &peer )==FD_FAILOVER_HELLO_OK ); + fd_memcpy( peer.staked_pubkey, public, 32UL ); + FD_TEST( fd_failover_hello_check( &ctx->channel->self_hello, &peer )==FD_FAILOVER_HELLO_ERR_STAKED ); + controller_fini( ctx ); +} + +/* test_standby_vote_counts: a vote our tower published counts toward + the floor whatever our role, set-identity can install the staked key + before we hear of it. */ +static void +test_standby_vote_counts( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_tower_slot_done_t done; + fd_memset( &done, 0, sizeof(done) ); + done.replay_slot = 130UL; + done.root_slot = FD_FAILOVER_SLOT_NULL; + done.has_vote_txn = 1; + done.vote_slot = 129UL; + consume_slot_done( ctx, &done ); + FD_TEST( ctx->own_floor==129UL && ctx->last_vote_slot==129UL ); + controller_fini( ctx ); +} + /* A request of ours bound to boot 77 of the member whose junk key is 0x22, as the tests' pair() makes it. */ static void @@ -402,9 +570,9 @@ test_wait_result_redials( void ) { static void test_refused_command_keeps_stuck( void ) { fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); - ctx->stuck = 1; + ctx->stuck = 1; + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS && ctx->stuck ); ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); - FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_DEMOTE, 0U, 0 )==FD_FAILOVER_CONTROL_RESULT_BAD_ROLE && ctx->stuck ); FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS && !ctx->stuck ); controller_fini( ctx ); } @@ -512,6 +680,9 @@ main( int argc, char ** argv ) { test_no_final_state(); test_request_replay_behind(); test_demote_drain(); + test_operator_switch(); + test_operator_failover_identity(); + test_standby_vote_counts(); test_result_wait_peer_restart(); test_session_change_drops_result(); test_request_address(); From b27f4a53abdfa6af64bc82f7f8bd5ce4c0f09ca7 Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Sun, 18 Oct 2026 15:16:05 +0000 Subject: [PATCH 13/23] Harden failover authentication and bound public listener work --- book/api/cli.md | 10 +- book/api/metrics-generated.md | 24 + src/app/firedancer/commands/failover.c | 2 + src/app/firedancer/topology.c | 10 +- src/app/shared/fd_config.c | 1 - src/app/shared/test_config_parse.c | 1 - src/disco/keyguard/fd_keyguard.h | 21 +- src/disco/keyguard/fd_keyguard_authorize.c | 10 +- src/disco/keyguard/fd_keyguard_match.c | 11 +- src/disco/keyguard/fd_sign_tile.c | 16 +- src/disco/keyguard/fuzz_keyguard.c | 17 +- src/disco/keyguard/test_keyguard.c | 12 +- src/disco/keyguard/test_sign_tile.c | 36 +- src/disco/metrics/generate/types.py | 1 + src/disco/metrics/generated/fd_metrics_all.c | 21 + src/disco/metrics/generated/fd_metrics_all.h | 3 +- .../metrics/generated/fd_metrics_enums.h | 10 + .../metrics/generated/fd_metrics_failov.h | 97 +++ .../metrics/generated/fd_metrics_sample_gen.c | 2 +- src/disco/metrics/metrics.xml | 22 + src/disco/stem/fd_stem.c | 7 +- src/disco/stem/fd_stem_run1.c | 3 + src/discof/admin/fd_adminctl.h | 1 + src/discof/failover/fd_failover_channel.c | 745 +++++++++++++----- src/discof/failover/fd_failover_channel.h | 67 +- src/discof/failover/fd_failover_proto.c | 51 +- src/discof/failover/fd_failover_proto.h | 74 +- src/discof/failover/fd_failover_tile.c | 158 ++-- src/discof/failover/fd_failover_tls.c | 102 ++- src/discof/failover/fd_failover_tls.h | 18 +- src/discof/failover/fd_failover_wire.c | 16 +- src/discof/failover/test_failover_channel.c | 15 +- src/discof/failover/test_failover_tile.c | 63 +- 33 files changed, 1222 insertions(+), 425 deletions(-) create mode 100644 src/disco/metrics/generated/fd_metrics_failov.h diff --git a/book/api/cli.md b/book/api/cli.md index 882dc7696ab..77b12c61f20 100644 --- a/book/api/cli.md +++ b/book/api/cli.md @@ -369,9 +369,10 @@ terminal unless `--yes` is given. Outside a terminal it proceeds without a prompt. Before asking for confirmation, the command checks the local validator's -role. The connection to the remote validator starts after confirmation, -so the prompt does not mean the remote validator is reachable or ready -to hand off. +role. A standby with a prepared target identity and a fresh gossip +address authenticates to the remote validator in advance. This connection +does not request a handoff. Confirmation submits the handoff request, so +the prompt does not mean the remote validator is ready to hand off. The local validator finds the active's address through gossip. Use `--address` to give the remote validator's IPv4 address or host name @@ -560,7 +561,8 @@ handoff progress. The following example shows a completed handoff. The last handoff ID matches the ID printed by `failover promote` above. The connection closes after completion, so `peer role` is unknown until another authenticated -session is established. +session is established. The new standby reconnects when it discovers the +new active through gossip. <<< @/snippets/commands/failover-status.ansi diff --git a/book/api/metrics-generated.md b/book/api/metrics-generated.md index 94a47a3b506..2996f107d99 100644 --- a/book/api/metrics-generated.md +++ b/book/api/metrics-generated.md @@ -1822,6 +1822,30 @@ +## Failov Tile + +
+ +| Metric | Type | Description | +|--------|------|-------------| +| failov_​conn_​accepted | counter | TCP connections accepted before admission checks | +| failov_​accept_​budget_​exhausted | counter | Polls that stopped accepting because the syscall budget was empty | +| failov_​conn_​rejected
{failover_​conn_​reject_​reason="source"} | counter | Accepted TCP connections refused before candidate allocation, by reason (A candidate from this address is open or its admission quota has not expired) | +| failov_​conn_​rejected
{failover_​conn_​reject_​reason="budget"} | counter | Accepted TCP connections refused before candidate allocation, by reason (The ordinary or reconnect admission budget is empty) | +| failov_​conn_​rejected
{failover_​conn_​reject_​reason="capacity"} | counter | Accepted TCP connections refused before candidate allocation, by reason (The corresponding handshake pool is full) | +| failov_​conn_​reconnect_​admitted | counter | Candidates admitted to the reserved lane for recently authenticated addresses | +| failov_​handshake_​timed_​out | counter | Candidates closed at a phase or absolute handshake deadline | +| failov_​proof_​rejected | counter | Admission proofs that failed target identity authentication | +| failov_​hello_​rejected | counter | Admission or TLS HELLO checks that failed | +| failov_​tls_​handshake_​failed | counter | TLS handshakes that failed after admission | +| failov_​frame_​protocol_​rejected | counter | Authenticated frames rejected by the controller | +| failov_​conn_​authenticated | counter | Completed mutual admission, TLS and HELLO authentications | +| failov_​handshake_​duration_​nanos | counter | Cumulative time from candidate allocation to completed authentication | +| failov_​handshake_​latest_​duration_​nanos | gauge | Duration of the most recently completed authentication | +| failov_​conn_​pending | gauge | Candidates waiting for admission, TLS or HELLO authentication | + +
+ ## Snapmk Tile
diff --git a/src/app/firedancer/commands/failover.c b/src/app/firedancer/commands/failover.c index 25225fbb1bf..c413f103f37 100644 --- a/src/app/firedancer/commands/failover.c +++ b/src/app/firedancer/commands/failover.c @@ -155,6 +155,7 @@ control_result_name( ulong result ) { case FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED: return "provide the active's keypair with `failover promote ` to prepare this client's target identity"; case FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT: return "the keypair must match the prepared target and must differ from this machine's boot identity"; case FD_FAILOVER_CONTROL_RESULT_NOT_BOOT_IDENTITY: return "the client must still be using its boot identity when preparing a target"; + case FD_FAILOVER_CONTROL_RESULT_INPUT_LOST: return "failover lost an input update and is disabled until restart, validator processing continues"; case FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER: return "the target identity is already an authorized voter, remove it from the authorized voters before preparing it"; case FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH: return "the public key in the keypair file does not match its private key"; default: return NULL; @@ -201,6 +202,7 @@ failover_confirm( args_t const * args ) { static char const * other_holder( ulong result ) { switch( result ) { + case FD_FAILOVER_CONTROL_RESULT_INPUT_LOST: return "unknown after input loss, failover requires a restart"; case FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED: return "no target identity prepared"; case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: return "n/a, this machine holds the identity"; case FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE: return "the authenticated peer says it holds the identity"; diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index f146e3c3dec..05ae973b7e0 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -783,16 +783,18 @@ fd_topo_initialize( config_t * config ) { if( !config->tiles.gui.enabled ) { fd_topob_tile_out( topo, "gossip", 0UL, "gossip_ciseen", 0UL ); } - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "gossip_ciseen", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /* contact infos for the peer's address */ + /* Failover cannot backpressure validator work. An input overrun + disables failover until restart instead of trusting stale state. */ + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "gossip_ciseen", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* contact infos for the peer's address */ /**/ fd_topob_tile_out( topo, "admin", 0UL, "admin_failov", 0UL ); - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "admin_failov", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "admin_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_admin", 0UL ); /**/ fd_topob_tile_in ( topo, "admin", 0UL, "metric_in", "failov_admin", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); } if( !alpenglow_enabled && failover_enabled ) { - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "failov", 0UL, "adopt_tower", 0UL ); - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_adopt", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_adopt", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); } FOR(execrp_tile_cnt) fd_topob_tile_in ( topo, "execrp", i, "metric_in", "replay_execrp", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); diff --git a/src/app/shared/fd_config.c b/src/app/shared/fd_config.c index 042360b9632..49089525e4b 100644 --- a/src/app/shared/fd_config.c +++ b/src/app/shared/fd_config.c @@ -555,7 +555,6 @@ fd_config_validate_failover( fd_config_t const * config ) { } } - void fd_config_validate( fd_config_t const * config ) { fd_config_validatef( &config->firedancer ); diff --git a/src/app/shared/test_config_parse.c b/src/app/shared/test_config_parse.c index d55ba72ddee..297f1e63c57 100644 --- a/src/app/shared/test_config_parse.c +++ b/src/app/shared/test_config_parse.c @@ -324,7 +324,6 @@ main( int argc, /* Parse the failover knobs */ fd_memset( config, 0, sizeof(config_t) ); - config->is_firedancer = 1; pod = fd_pod_join( fd_pod_new( pod_mem, sizeof(pod_mem) ) ); FD_TEST( fd_toml_parse( cfg_str_failover, sizeof(cfg_str_failover)-1, pod, scratch, sizeof(scratch), NULL ) == FD_TOML_SUCCESS ); FD_TEST( fd_config_extract_pod( pod, config ) == config ); diff --git a/src/disco/keyguard/fd_keyguard.h b/src/disco/keyguard/fd_keyguard.h index 2b20cbd9c8e..046dbc995d9 100644 --- a/src/disco/keyguard/fd_keyguard.h +++ b/src/disco/keyguard/fd_keyguard.h @@ -36,19 +36,16 @@ FD_STATIC_ASSERT( FD_KEYGUARD_AUTH_VOTERS_MAX<=16UL, auth_voters_max_fits_sign_r #define FD_KEYGUARD_ROLE_RSERVE (8) /* Repair server tile */ #define FD_KEYGUARD_ROLE_VOTOR (9) /* Alpenglow votor tile (QUIC TLS, vote history file) */ #define FD_KEYGUARD_ROLE_TOWER (10) /* Tower tile (tower file) */ -#define FD_KEYGUARD_ROLE_FAILOV (11) /* Failover tile member certificate and TLS */ +#define FD_KEYGUARD_ROLE_FAILOV (11) /* Failover admission proof and TLS */ #define FD_KEYGUARD_ROLE_CNT (12) /* number of known roles */ -/* The failov tile gets its member certificate signed, a plain Ed25519 - signature by the failover identity over a 16 byte domain prefix - followed by - the sign tile's own junk pubkey. The prefix keeps that signature out - of every other raw message domain, in particular the 32 byte shred - roots the leader role signs. The junk key signs its TLS - CertificateVerify. */ -#define FD_KEYGUARD_MEMBER_CERT_PREFIX "FD_FAILOVER_MBR1" -#define FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ (16UL) -#define FD_KEYGUARD_MEMBER_CERT_MSG_SZ (48UL) +/* Admission proofs use the prepared failover key. Each proof covers a + role-specific domain, our own junk pubkey and a transcript hash. + TLS CertificateVerify uses the junk key. */ +#define FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN "FD_FAILOVER_CLI1" +#define FD_KEYGUARD_FAILOVER_SERVER_DOMAIN "FD_FAILOVER_SRV1" +#define FD_KEYGUARD_FAILOVER_DOMAIN_SZ (16UL) +#define FD_KEYGUARD_FAILOVER_PROOF_SZ (FD_KEYGUARD_FAILOVER_DOMAIN_SZ+32UL+32UL) /* Payload types ******************************************************/ @@ -66,7 +63,7 @@ FD_STATIC_ASSERT( FD_KEYGUARD_AUTH_VOTERS_MAX<=16UL, auth_voters_max_fits_sign_r #define FD_KEYGUARD_PAYLOAD_LG_BLS_PUBKEY (12) /* Alpenglow BLS public key query */ #define FD_KEYGUARD_PAYLOAD_LG_TOWER (13) /* Tower file */ #define FD_KEYGUARD_PAYLOAD_LG_VOTE_HISTORY (14) /* Alpenglow vote history file */ -#define FD_KEYGUARD_PAYLOAD_LG_FAILOV (15) /* Failover member certificate */ +#define FD_KEYGUARD_PAYLOAD_LG_FAILOV (15) /* Failover admission proof */ #define FD_KEYGUARD_PAYLOAD_TXN (1UL<_data, sz, FD_KEYGUARD_SIGN_TYPE_ED25519 ); + if( payload==FD_KEYGUARD_PAYLOAD_TLS_CV ) { fd_ed25519_sign( dst, ctx->_data, sz, ctx->failover_junk_key+32UL, ctx->failover_junk_key, ctx->sha512 ); return; } - if( FD_UNLIKELY( !fd_memeq( ctx->_data+FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ, ctx->failover_junk_key+32UL, 32UL ) ) ) { - FD_LOG_EMERG(( "member certificate request is not for our junk key" )); + if( FD_UNLIKELY( payload!=FD_KEYGUARD_PAYLOAD_FAILOV ) ) FD_LOG_EMERG(( "invalid failover signing domain" )); + if( FD_UNLIKELY( !fd_memeq( ctx->_data+FD_KEYGUARD_FAILOVER_DOMAIN_SZ, ctx->failover_junk_key+32UL, 32UL ) ) ) { + FD_LOG_EMERG(( "admission proof request is not for our junk key" )); } - if( FD_UNLIKELY( !ctx->failover_key ) ) FD_LOG_EMERG(( "member certificate requested before preparing a failover identity" )); + if( FD_UNLIKELY( !ctx->failover_key ) ) FD_LOG_EMERG(( "admission proof requested before preparing a failover identity" )); fd_ed25519_sign( dst, ctx->_data, sz, ctx->failover_key+32UL, ctx->failover_key, ctx->sha512 ); } diff --git a/src/disco/keyguard/fuzz_keyguard.c b/src/disco/keyguard/fuzz_keyguard.c index 18eace0288b..814591a9967 100644 --- a/src/disco/keyguard/fuzz_keyguard.c +++ b/src/disco/keyguard/fuzz_keyguard.c @@ -31,16 +31,17 @@ LLVMFuzzerTestOneInput( uchar const * data, } } - /* The sign tile picks the failover key by size, so the failov role - signs only the member certificate or a CertificateVerify. */ + /* The sign tile checks the failover signing domain, so the failov role + signs only the admission proof or a CertificateVerify. */ for( ulong i=0UL; ipublic_key, FD_BLS_PUB_COMPRESSED_SZ ) ); - FD_TEST( failover_signed_by( cert, sizeof(cert), staked_key+32UL ) ); + FD_TEST( failover_signed_by( proof, sizeof(proof), staked_key+32UL ) ); FD_TEST( failover_signed_by( cv, sizeof(cv), junk_key +32UL ) ); } @@ -437,7 +437,7 @@ test_failover_keys( void ) { } /* A set-identity keypair other than the junk key becomes the failover - identity, it signs the member certificate and cannot become an + identity, it signs the admission proof and cannot become an authorized voter. The staked key keeps its boot page, another key goes in the operator page. */ @@ -460,16 +460,16 @@ test_failover_operator_key( void ) { uchar third[ 64 ]; fd_memset( third, 0x66, 32UL ); fd_ed25519_public_from_private( third+32UL, third, ctx.sha512 ); - uchar cert[ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ]; - fd_memcpy( cert, FD_KEYGUARD_MEMBER_CERT_PREFIX, FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ ); - fd_memcpy( cert+FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ, junk_key+32UL, 32UL ); + uchar proof[ FD_KEYGUARD_FAILOVER_PROOF_SZ ] = {0}; + fd_memcpy( proof, FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN, FD_KEYGUARD_FAILOVER_DOMAIN_SZ ); + fd_memcpy( proof+FD_KEYGUARD_FAILOVER_DOMAIN_SZ, junk_key+32UL, 32UL ); uchar staked_copy[ 64 ]; fd_memcpy( staked_copy, staked_key, 64UL ); operator_switch( identity_switch, third ); FD_TEST( fd_memeq( ctx.public_key, third+32UL, 32UL ) && fd_memeq( operator_key, third, 64UL ) ); FD_TEST( ctx.failover_key==operator_key && fd_memeq( staked_key, staked_copy, 64UL ) ); - FD_TEST( failover_signed_by( cert, sizeof(cert), third+32UL ) ); + FD_TEST( failover_signed_by( proof, sizeof(proof), third+32UL ) ); /* A failover switch can now select the new key and the junk key. */ failover_switch( identity_switch, junk_key+32UL ); @@ -480,7 +480,7 @@ test_failover_operator_key( void ) { /* set-identity to the junk key keeps the failover identity. */ operator_switch( identity_switch, junk_key ); FD_TEST( fd_memeq( ctx.public_key, junk_key+32UL, 32UL ) && ctx.failover_key==operator_key && fd_memeq( operator_key, third, 64UL ) ); - FD_TEST( failover_signed_by( cert, sizeof(cert), third+32UL ) ); + FD_TEST( failover_signed_by( proof, sizeof(proof), third+32UL ) ); /* The failover identity cannot become an authorized voter, the configured key may be one now. */ @@ -506,7 +506,7 @@ test_failover_operator_key( void ) { operator_switch( identity_switch, staked_key ); uchar zero[ 64 ] = {0}; FD_TEST( ctx.failover_key==staked_key && fd_memeq( operator_key, zero, 64UL ) ); - FD_TEST( failover_signed_by( cert, sizeof(cert), staked_key+32UL ) ); + FD_TEST( failover_signed_by( proof, sizeof(proof), staked_key+32UL ) ); failover_switch( identity_switch, junk_key+32UL ); failover_switch( identity_switch, staked_key+32UL ); FD_TEST( fd_memeq( ctx.public_key, staked_key+32UL, 32UL ) ); @@ -524,10 +524,10 @@ switch_to_unknown_key( void ) { } static void -sign_foreign_cert( void ) { - fd_memcpy( ctx._data, FD_KEYGUARD_MEMBER_CERT_PREFIX, FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ ); - fd_memcpy( ctx._data+FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ, staked_key+32UL, 32UL ); - after_frag( &ctx, 0UL, 0UL, FD_KEYGUARD_SIGN_TYPE_ED25519, FD_KEYGUARD_MEMBER_CERT_MSG_SZ, 0UL, 0UL, &stem ); +sign_foreign_proof( void ) { + fd_memcpy( ctx._data, FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN, FD_KEYGUARD_FAILOVER_DOMAIN_SZ ); + fd_memcpy( ctx._data+FD_KEYGUARD_FAILOVER_DOMAIN_SZ, staked_key+32UL, 32UL ); + after_frag( &ctx, 0UL, 0UL, FD_KEYGUARD_SIGN_TYPE_ED25519, FD_KEYGUARD_FAILOVER_PROOF_SZ, 0UL, 0UL, &stem ); } static int @@ -546,7 +546,7 @@ child_exit_status( void (*fn)( void ) ) { } /* A switch to a key that is neither the junk key nor the failover - identity, and a certificate over another junk key, stop the + identity, and a proof over another junk key, stop the validator. */ static void @@ -555,7 +555,7 @@ test_failover_refusals( void ) { ctx.keyswitch = fd_keyswitch_join( fd_keyswitch_new( failover_ks, FD_KEYSWITCH_STATE_UNLOCKED ) ); FD_TEST( ctx.keyswitch ); FD_TEST( child_exit_status( switch_to_unknown_key )==1 ); - FD_TEST( child_exit_status( sign_foreign_cert )==1 ); + FD_TEST( child_exit_status( sign_foreign_proof )==1 ); FD_LOG_NOTICE(( "pass: failover refusals" )); } diff --git a/src/disco/metrics/generate/types.py b/src/disco/metrics/generate/types.py index 43baa73b277..07e03bbf9e5 100644 --- a/src/disco/metrics/generate/types.py +++ b/src/disco/metrics/generate/types.py @@ -47,6 +47,7 @@ class Tile(Enum): MLX5 = 41 WAKER = 42 MWAITX = 43 + FAILOV = 44 SNAPMK = 50 SNAPZP = 51 diff --git a/src/disco/metrics/generated/fd_metrics_all.c b/src/disco/metrics/generated/fd_metrics_all.c index d494ab83a18..8388a5aff15 100644 --- a/src/disco/metrics/generated/fd_metrics_all.c +++ b/src/disco/metrics/generated/fd_metrics_all.c @@ -1607,6 +1607,24 @@ const fd_metrics_meta_t FD_METRICS_MWAITX[FD_METRICS_MWAITX_TOTAL] = { DECLARE_METRIC_ENUM( MWAITX_WAKE_ISSUED, COUNTER, MWAITX_WAKE_CAUSE, SWEEP ), }; +const fd_metrics_meta_t FD_METRICS_FAILOV[FD_METRICS_FAILOV_TOTAL] = { + DECLARE_METRIC( FAILOV_CONN_ACCEPTED, COUNTER ), + DECLARE_METRIC( FAILOV_ACCEPT_BUDGET_EXHAUSTED, COUNTER ), + DECLARE_METRIC_ENUM( FAILOV_CONN_REJECTED, COUNTER, FAILOVER_CONN_REJECT_REASON, SOURCE ), + DECLARE_METRIC_ENUM( FAILOV_CONN_REJECTED, COUNTER, FAILOVER_CONN_REJECT_REASON, BUDGET ), + DECLARE_METRIC_ENUM( FAILOV_CONN_REJECTED, COUNTER, FAILOVER_CONN_REJECT_REASON, CAPACITY ), + DECLARE_METRIC( FAILOV_CONN_RECONNECT_ADMITTED, COUNTER ), + DECLARE_METRIC( FAILOV_HANDSHAKE_TIMED_OUT, COUNTER ), + DECLARE_METRIC( FAILOV_PROOF_REJECTED, COUNTER ), + DECLARE_METRIC( FAILOV_HELLO_REJECTED, COUNTER ), + DECLARE_METRIC( FAILOV_TLS_HANDSHAKE_FAILED, COUNTER ), + DECLARE_METRIC( FAILOV_FRAME_PROTOCOL_REJECTED, COUNTER ), + DECLARE_METRIC( FAILOV_CONN_AUTHENTICATED, COUNTER ), + DECLARE_METRIC( FAILOV_HANDSHAKE_DURATION_NANOS, COUNTER ), + DECLARE_METRIC( FAILOV_HANDSHAKE_LATEST_DURATION_NANOS, GAUGE ), + DECLARE_METRIC( FAILOV_CONN_PENDING, GAUGE ), +}; + const fd_metrics_meta_t FD_METRICS_SNAPMK[FD_METRICS_SNAPMK_TOTAL] = { DECLARE_METRIC( SNAPMK_JOB_STATE, GAUGE ), DECLARE_METRIC_ENUM( SNAPMK_SNAPSHOT_STARTED, COUNTER, SNAP_TYPE, FULL ), @@ -1968,6 +1986,7 @@ const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT] = { "mlx5", "waker", "mwaitx", + "failov", "snapmk", "snapzp", "snaprd", @@ -2014,6 +2033,7 @@ const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = { FD_METRICS_MLX5_TOTAL, FD_METRICS_WAKER_TOTAL, FD_METRICS_MWAITX_TOTAL, + FD_METRICS_FAILOV_TOTAL, FD_METRICS_SNAPMK_TOTAL, FD_METRICS_SNAPZP_TOTAL, FD_METRICS_SNAPRD_TOTAL, @@ -2059,6 +2079,7 @@ const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT] FD_METRICS_MLX5, FD_METRICS_WAKER, FD_METRICS_MWAITX, + FD_METRICS_FAILOV, FD_METRICS_SNAPMK, FD_METRICS_SNAPZP, FD_METRICS_SNAPRD, diff --git a/src/disco/metrics/generated/fd_metrics_all.h b/src/disco/metrics/generated/fd_metrics_all.h index 7481258cadf..be89f95d595 100644 --- a/src/disco/metrics/generated/fd_metrics_all.h +++ b/src/disco/metrics/generated/fd_metrics_all.h @@ -90,6 +90,7 @@ enum { #include "fd_metrics_admin.h" #include "fd_metrics_waker.h" #include "fd_metrics_mwaitx.h" +#include "fd_metrics_failov.h" #include "fd_metrics_ipecho.h" #include "fd_metrics_backt.h" #include "fd_metrics_execrp.h" @@ -250,7 +251,7 @@ extern const fd_metrics_meta_t FD_METRICS_ALL_LINK_IN[FD_METRICS_ALL_LINK_IN_TOT #define FD_METRICS_TOTAL_SZ (8UL*301UL) -#define FD_METRICS_TILE_KIND_CNT 43 +#define FD_METRICS_TILE_KIND_CNT 44 extern const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT]; extern const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT]; extern const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT]; diff --git a/src/disco/metrics/generated/fd_metrics_enums.h b/src/disco/metrics/generated/fd_metrics_enums.h index e53fc89ca94..cc3e7e44494 100644 --- a/src/disco/metrics/generated/fd_metrics_enums.h +++ b/src/disco/metrics/generated/fd_metrics_enums.h @@ -1337,6 +1337,16 @@ #define FD_METRICS_ENUM_PORT_CHECK_TCP_RESULT_V_EVICTED_NAME "evicted" #define FD_METRICS_ENUM_PORT_CHECK_TCP_RESULT_G_ALL_MASK (0x3fUL) +#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_NAME "failover_conn_reject_reason" +#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_CNT (3UL) +#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_SOURCE_IDX 0 +#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_SOURCE_NAME "source" +#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_BUDGET_IDX 1 +#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_BUDGET_NAME "budget" +#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_CAPACITY_IDX 2 +#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_CAPACITY_NAME "capacity" +#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_G_ALL_MASK (0x7UL) + #define FD_METRICS_ENUM_TXN_REGIME_NAME "txn_regime" #define FD_METRICS_ENUM_TXN_REGIME_CNT (3UL) #define FD_METRICS_ENUM_TXN_REGIME_V_LOAD_IDX 0 diff --git a/src/disco/metrics/generated/fd_metrics_failov.h b/src/disco/metrics/generated/fd_metrics_failov.h new file mode 100644 index 00000000000..3adf1ca11a5 --- /dev/null +++ b/src/disco/metrics/generated/fd_metrics_failov.h @@ -0,0 +1,97 @@ +#ifndef HEADER_fd_src_disco_metrics_generated_fd_metrics_failov_h +#define HEADER_fd_src_disco_metrics_generated_fd_metrics_failov_h + +/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */ + +#include "../fd_metrics_base.h" +#include "fd_metrics_enums.h" + +enum { + FD_METRICS_COUNTER_FAILOV_CONN_ACCEPTED_OFF = FD_METRICS_TILE_OFF, + FD_METRICS_COUNTER_FAILOV_ACCEPT_BUDGET_EXHAUSTED_OFF, + FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_OFF, + FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_SOURCE_OFF = FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_OFF, + FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_BUDGET_OFF, + FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_CAPACITY_OFF, + FD_METRICS_COUNTER_FAILOV_CONN_RECONNECT_ADMITTED_OFF, + FD_METRICS_COUNTER_FAILOV_HANDSHAKE_TIMED_OUT_OFF, + FD_METRICS_COUNTER_FAILOV_PROOF_REJECTED_OFF, + FD_METRICS_COUNTER_FAILOV_HELLO_REJECTED_OFF, + FD_METRICS_COUNTER_FAILOV_TLS_HANDSHAKE_FAILED_OFF, + FD_METRICS_COUNTER_FAILOV_FRAME_PROTOCOL_REJECTED_OFF, + FD_METRICS_COUNTER_FAILOV_CONN_AUTHENTICATED_OFF, + FD_METRICS_COUNTER_FAILOV_HANDSHAKE_DURATION_NANOS_OFF, + FD_METRICS_GAUGE_FAILOV_HANDSHAKE_LATEST_DURATION_NANOS_OFF, + FD_METRICS_GAUGE_FAILOV_CONN_PENDING_OFF, +}; + +#define FD_METRICS_COUNTER_FAILOV_CONN_ACCEPTED_NAME "failov_conn_accepted" +#define FD_METRICS_COUNTER_FAILOV_CONN_ACCEPTED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_CONN_ACCEPTED_DESC "TCP connections accepted before admission checks" +#define FD_METRICS_COUNTER_FAILOV_CONN_ACCEPTED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_FAILOV_ACCEPT_BUDGET_EXHAUSTED_NAME "failov_accept_budget_exhausted" +#define FD_METRICS_COUNTER_FAILOV_ACCEPT_BUDGET_EXHAUSTED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_ACCEPT_BUDGET_EXHAUSTED_DESC "Polls that stopped accepting because the syscall budget was empty" +#define FD_METRICS_COUNTER_FAILOV_ACCEPT_BUDGET_EXHAUSTED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_NAME "failov_conn_rejected" +#define FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_DESC "Accepted TCP connections refused before candidate allocation, by reason" +#define FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_CVT (FD_METRICS_CONVERTER_NONE) +#define FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_CNT (3UL) + +#define FD_METRICS_COUNTER_FAILOV_CONN_RECONNECT_ADMITTED_NAME "failov_conn_reconnect_admitted" +#define FD_METRICS_COUNTER_FAILOV_CONN_RECONNECT_ADMITTED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_CONN_RECONNECT_ADMITTED_DESC "Candidates admitted to the reserved lane for recently authenticated addresses" +#define FD_METRICS_COUNTER_FAILOV_CONN_RECONNECT_ADMITTED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_TIMED_OUT_NAME "failov_handshake_timed_out" +#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_TIMED_OUT_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_TIMED_OUT_DESC "Candidates closed at a phase or absolute handshake deadline" +#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_TIMED_OUT_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_FAILOV_PROOF_REJECTED_NAME "failov_proof_rejected" +#define FD_METRICS_COUNTER_FAILOV_PROOF_REJECTED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_PROOF_REJECTED_DESC "Admission proofs that failed target identity authentication" +#define FD_METRICS_COUNTER_FAILOV_PROOF_REJECTED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_FAILOV_HELLO_REJECTED_NAME "failov_hello_rejected" +#define FD_METRICS_COUNTER_FAILOV_HELLO_REJECTED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_HELLO_REJECTED_DESC "Admission or TLS HELLO checks that failed" +#define FD_METRICS_COUNTER_FAILOV_HELLO_REJECTED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_FAILOV_TLS_HANDSHAKE_FAILED_NAME "failov_tls_handshake_failed" +#define FD_METRICS_COUNTER_FAILOV_TLS_HANDSHAKE_FAILED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_TLS_HANDSHAKE_FAILED_DESC "TLS handshakes that failed after admission" +#define FD_METRICS_COUNTER_FAILOV_TLS_HANDSHAKE_FAILED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_FAILOV_FRAME_PROTOCOL_REJECTED_NAME "failov_frame_protocol_rejected" +#define FD_METRICS_COUNTER_FAILOV_FRAME_PROTOCOL_REJECTED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_FRAME_PROTOCOL_REJECTED_DESC "Authenticated frames rejected by the controller" +#define FD_METRICS_COUNTER_FAILOV_FRAME_PROTOCOL_REJECTED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_FAILOV_CONN_AUTHENTICATED_NAME "failov_conn_authenticated" +#define FD_METRICS_COUNTER_FAILOV_CONN_AUTHENTICATED_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_CONN_AUTHENTICATED_DESC "Completed mutual admission, TLS and HELLO authentications" +#define FD_METRICS_COUNTER_FAILOV_CONN_AUTHENTICATED_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_DURATION_NANOS_NAME "failov_handshake_duration_nanos" +#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_DURATION_NANOS_TYPE (FD_METRICS_TYPE_COUNTER) +#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_DURATION_NANOS_DESC "Cumulative time from candidate allocation to completed authentication" +#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_GAUGE_FAILOV_HANDSHAKE_LATEST_DURATION_NANOS_NAME "failov_handshake_latest_duration_nanos" +#define FD_METRICS_GAUGE_FAILOV_HANDSHAKE_LATEST_DURATION_NANOS_TYPE (FD_METRICS_TYPE_GAUGE) +#define FD_METRICS_GAUGE_FAILOV_HANDSHAKE_LATEST_DURATION_NANOS_DESC "Duration of the most recently completed authentication" +#define FD_METRICS_GAUGE_FAILOV_HANDSHAKE_LATEST_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_GAUGE_FAILOV_CONN_PENDING_NAME "failov_conn_pending" +#define FD_METRICS_GAUGE_FAILOV_CONN_PENDING_TYPE (FD_METRICS_TYPE_GAUGE) +#define FD_METRICS_GAUGE_FAILOV_CONN_PENDING_DESC "Candidates waiting for admission, TLS or HELLO authentication" +#define FD_METRICS_GAUGE_FAILOV_CONN_PENDING_CVT (FD_METRICS_CONVERTER_NONE) + +#define FD_METRICS_FAILOV_TOTAL (15UL) +extern const fd_metrics_meta_t FD_METRICS_FAILOV[FD_METRICS_FAILOV_TOTAL]; + +#endif /* HEADER_fd_src_disco_metrics_generated_fd_metrics_failov_h */ diff --git a/src/disco/metrics/generated/fd_metrics_sample_gen.c b/src/disco/metrics/generated/fd_metrics_sample_gen.c index aae1ea0db5c..6f34422bcb9 100644 --- a/src/disco/metrics/generated/fd_metrics_sample_gen.c +++ b/src/disco/metrics/generated/fd_metrics_sample_gen.c @@ -4401,6 +4401,6 @@ fd_metrics_sample_schema_t const fd_metrics_sample_tiles[] = { { NULL, 0U, NULL, 0UL, 0UL } }; -char const * const fd_metrics_sample_skip[] = { "backt", "benchs", "solcap", NULL }; +char const * const fd_metrics_sample_skip[] = { "backt", "benchs", "failov", "solcap", NULL }; fd_metrics_sample_schema_t const fd_metrics_sample_link = { "link", 61U, fields_link, 9UL, 284UL }; diff --git a/src/disco/metrics/metrics.xml b/src/disco/metrics/metrics.xml index 269b0fd4e9a..c012413e131 100644 --- a/src/disco/metrics/metrics.xml +++ b/src/disco/metrics/metrics.xml @@ -1664,6 +1664,28 @@ deliberate exception with lint-allow="") + + + + + + + + + + + + + + + + + + + + + + diff --git a/src/disco/stem/fd_stem.c b/src/disco/stem/fd_stem.c index 04156cacff2..748fb04ec62 100644 --- a/src/disco/stem/fd_stem.c +++ b/src/disco/stem/fd_stem.c @@ -169,7 +169,11 @@ AFTER_POLL_OVERRUN Is called when an overrun is detected while polling for new frags. This callback is not called when an overrun is detected in - during_frag. */ + during_frag. + + AFTER_READ_OVERRUN + Is called when an overrun is detected after during_frag. after_frag + is not called for the overwritten fragment. */ #include "../../util/log/fd_log.h" #include "../topo/fd_topo.h" @@ -817,3 +821,4 @@ STEM_(run)( fd_topo_t * topo, #undef STEM_CALLBACK_RETURNABLE_FRAG #undef STEM_CALLBACK_AFTER_FRAG #undef STEM_CALLBACK_AFTER_POLL_OVERRUN +#undef STEM_CALLBACK_AFTER_READ_OVERRUN diff --git a/src/disco/stem/fd_stem_run1.c b/src/disco/stem/fd_stem_run1.c index 1a37258d5d7..0b8ea73384d 100644 --- a/src/disco/stem/fd_stem_run1.c +++ b/src/disco/stem/fd_stem_run1.c @@ -594,6 +594,9 @@ STEM_(STEM_RUN1_NAME)( ulong in_cnt, FD_COMPILER_MFENCE(); if( FD_UNLIKELY( fd_seq_ne( seq_test, seq_found ) ) ) { /* Overrun while reading (impossible if this_in honoring our fctl) */ +#ifdef STEM_CALLBACK_AFTER_READ_OVERRUN + STEM_CALLBACK_AFTER_READ_OVERRUN( ctx ); +#endif this_in->seq = seq_test; /* Resume from here (probably reasonably current, could query in mcache sync instead) */ fd_metrics_link_in( fd_metrics_base_tl, this_in->idx )[ FD_METRICS_COUNTER_LINK_LINK_READING_OVERRUN_OFF ]++; /* No local accum since extremely rare, faster to use smaller cache line */ fd_metrics_link_in( fd_metrics_base_tl, this_in->idx )[ FD_METRICS_COUNTER_LINK_FRAG_READING_OVERRUN_OFF ] += (uint)fd_seq_diff( seq_test, seq_found ); /* No local accum since extremely rare, faster to use smaller cache line */ diff --git a/src/discof/admin/fd_adminctl.h b/src/discof/admin/fd_adminctl.h index d0e1ece8c78..9c44d4471f1 100644 --- a/src/discof/admin/fd_adminctl.h +++ b/src/discof/admin/fd_adminctl.h @@ -83,6 +83,7 @@ #define FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED (0x4011UL) /* client has no target identity */ #define FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT (0x4012UL) /* target differs from the prepared identity or is the boot identity */ #define FD_FAILOVER_CONTROL_RESULT_NOT_BOOT_IDENTITY (0x4013UL) /* client must prepare while still on its boot identity */ +#define FD_FAILOVER_CONTROL_RESULT_INPUT_LOST (0x4014UL) /* failover input overrun, restart required */ struct fd_adminctl_add_auth_voter_v1 { ulong version; /* ==FD_ADMINCTL_ADD_AUTH_VOTER_PAYLOAD_VERSION */ diff --git a/src/discof/failover/fd_failover_channel.c b/src/discof/failover/fd_failover_channel.c index 054c3f725b6..4b2fc61b051 100644 --- a/src/discof/failover/fd_failover_channel.c +++ b/src/discof/failover/fd_failover_channel.c @@ -4,6 +4,8 @@ #include "../../ballet/base58/fd_base58.h" #include "../../util/fd_util.h" #include "../../util/net/fd_ip4.h" +#include "../../ballet/ed25519/fd_ed25519.h" +#include "../../waltz/tls/fd_tls_proto.h" #include #include @@ -11,71 +13,127 @@ #include #include -#define SOURCE_CNT (256UL) -#define PHASE_CONNECT (0) /* Wait for outbound TCP to connect. */ -#define PHASE_TLS (1) /* Complete the TLS handshake. */ -#define PHASE_HELLO (2) /* Read and validate the peer HELLO. */ -#define PHASE_READY (3) /* Drain our HELLO before pairing. */ +#define SOURCE_CNT (4096UL) +#define OUTBOUND_IDX (0UL) +#define ADMITTED_IDX (1UL) +#define RECONNECT_IDX (2UL) +#define UNAUTHENTICATED_IDX (3UL) +#define KNOWN_SOURCE_CNT (4UL) +#define PHASE_CONNECT (0) +#define PHASE_START_SEND (1) +#define PHASE_START_RECV (2) +#define PHASE_CHALLENGE_SEND (3) +#define PHASE_CHALLENGE_RECV (4) +#define PHASE_REQUEST_SIGN (5) +#define PHASE_REQUEST_SEND (6) +#define PHASE_REQUEST_RECV (7) +#define PHASE_RESPONSE_SIGN (8) +#define PHASE_RESPONSE_SEND (9) +#define PHASE_RESPONSE_RECV (10) +#define PHASE_TLS_START (11) +#define PHASE_TLS (12) +#define PHASE_HELLO (13) +#define PHASE_READY (14) + +FD_STATIC_ASSERT( FD_FAILOVER_CHANNEL_CANDIDATE_MAX<=8UL*sizeof(ulong), candidate_service_mask ); +FD_STATIC_ASSERT( FD_FAILOVER_CHANNEL_HELLO_TIMEOUT_NANOS<=FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS, source_initial_quota ); +FD_STATIC_ASSERT( SOURCE_CNT>=FD_FAILOVER_ADMIT_BURST+ + FD_FAILOVER_RECONNECT_BURST+ + ((FD_FAILOVER_ADMIT_RATE+FD_FAILOVER_RECONNECT_RATE)*2UL*(ulong)FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS+FD_FAILOVER_NANOS_PER_SECOND-1UL)/FD_FAILOVER_NANOS_PER_SECOND, + source_quota_capacity ); +FD_STATIC_ASSERT( FD_FAILOVER_CHANNEL_CANDIDATE_MAX>UNAUTHENTICATED_IDX, admission_pool_capacity ); + +FD_STATIC_ASSERT( FD_KEYGUARD_FAILOVER_PROOF_SZ<=FD_TLS_CV_SIGN_SZ, admission_sign_mtu ); +FD_STATIC_ASSERT( FD_FAILOVER_CLIENT_HELLO_MAX+sizeof(fd_tlsrec_hdr_t)+sizeof(fd_failover_admission_start_t)<=FD_FAILOVER_ADMISSION_MAX, admission_buffer ); /* A HELLO whose junk key does not match the key TLS authenticated */ #define HELLO_ERR_TLS_KEY (-1) struct candidate { - int fd; /* -1 marks an unused candidate. */ - int phase; - int dialed; - uint address; - long deadline; - fd_failover_tls_t tls; - fd_failover_wire_session_t wire; - fd_failover_hello_t hello; - uchar rx[ FD_FAILOVER_FRAME_MAX ]; - uchar tx[ FD_FAILOVER_FRAME_MAX ]; - ulong rx_used; - ulong tx_used; - ulong tx_sent; + int fd; /* -1 marks an unused candidate. */ + int phase; + int dialed; + int priority; /* Reserved crypto budget, never an authentication bypass. */ + uint address; + long deadline; + long admission_until; + long accepted_at; + int tls_started; + fd_failover_hello_t local_hello; + fd_failover_admission_start_t start; + fd_failover_admission_challenge_t challenge; + uchar client_hello[ FD_FAILOVER_CLIENT_HELLO_MAX+sizeof(fd_tlsrec_hdr_t) ]; + uchar transcript[ 32 ]; + uchar client_proof[ 64 ]; + fd_failover_tls_t tls; + fd_failover_wire_session_t wire; + fd_failover_hello_t hello; + uchar rx[ FD_FAILOVER_FRAME_MAX ]; + uchar tx[ FD_FAILOVER_FRAME_MAX ]; + ulong rx_used; + ulong tx_used; + ulong tx_sent; }; struct source { uint address; int used; - long updated; - ulong credit; + long last_admit; + long next_admit; + long backoff; +}; + +struct known_source { + uint address; + long until; }; struct fd_failover_channel { - ulong magic; /* FD_FAILOVER_CHANNEL_MAGIC once initialized */ - int dial_peer; /* the session machine runs as the dialer */ - ulong state; /* session state, stepped by fd_failover_session_step */ - uint peer_addr; /* the address a handoff dials, zero when we do not dial */ - ushort peer_port; /* the port a handoff dials */ - int listen_fd; /* listening socket, -1 until it is open */ - int paired_idx; /* the paired candidate, -1 while none is paired */ - ulong cursor; /* the next candidate to service, round robin */ - fd_failover_tls_ctx_t tls_ctx; /* TLS template every connection starts from */ - fd_failover_hello_t self_hello; /* the HELLO we send */ - fd_failover_hello_t peer_hello; /* the paired peer's HELLO, zero while unpaired */ - int member_cert_set; /* self_hello carries a checked member certificate */ - fd_sha512_t sha[ 1 ]; /* scratch for checking member certificates */ - struct candidate candidates[ FD_FAILOVER_CHANNEL_CANDIDATE_MAX ]; /* connections that are handshaking or paired */ - struct source sources[ SOURCE_CNT ]; /* accept rate limit per source address */ - long rate_updated; /* when rate_credit was last refilled */ - ulong rate_credit; /* accept rate limit across all addresses */ - long hello_timeout; /* how long a candidate has to pair */ - long silence_timeout; /* a paired session silent for this long is dropped */ - long last_rx; /* when the paired session last received a frame */ - long retry_at; /* when the dialer dials again */ - long paired_at; /* when the current session paired */ - long backoff; /* the current dial backoff, doubles up to backoff_max */ - long backoff_min; /* the shortest dial backoff, also the jitter range */ - long backoff_max; /* the longest dial backoff */ - long dial_log_at[ 3 ]; /* connect, early close, handshake timeout */ - long tls_log_at[ 4 ]; /* transport, ALPN, authentication, other */ - long hello_log_at[ 14 ]; /* one bounded bucket per HELLO refusal */ - long loss_log_at[ 6 ]; /* transport, silence, TLS, protocol, framing, local encode */ - long pair_log_at; /* when a pairing may be logged again */ - int expected_close; /* RESULT was sent, peer may close normally */ - ulong generation; /* identifies the current authenticated session */ + ulong magic; /* FD_FAILOVER_CHANNEL_MAGIC once initialized */ + int dial_peer; /* the session machine runs as the dialer */ + ulong state; /* session state, stepped by fd_failover_session_step */ + uint peer_addr; /* the dial destination, zero when we do not dial */ + ushort peer_port; /* the dial destination port */ + int listen_fd; /* listening socket, -1 until it is open */ + int paired_idx; /* the paired candidate, -1 while none is paired */ + ulong cursor; /* the next candidate to service, round robin */ + ulong proof_cursor; /* the next complete proof to verify, round robin */ + fd_failover_tls_ctx_t tls_ctx; /* TLS template every connection starts from */ + fd_failover_hello_t self_hello; /* the HELLO we send */ + fd_failover_hello_t peer_hello; /* the paired peer's HELLO, zero while unpaired */ + fd_failover_proof_signer_t proof_signer; + long verify_updated; + ulong verify_credit; + long priority_verify_updated; + ulong priority_verify_credit; + int verify_budget; + fd_sha512_t sha[ 1 ]; /* scratch for checking admission proofs */ + struct candidate candidates[ FD_FAILOVER_CHANNEL_CANDIDATE_MAX ]; /* connections that are handshaking or paired */ + struct source sources[ SOURCE_CNT ]; /* accept rate limit per source address */ + struct known_source known_sources[ KNOWN_SOURCE_CNT ]; /* authenticated addresses, expiring */ + fd_failover_channel_metrics_t metrics; + long rate_updated; /* when rate_credit was last refilled */ + ulong rate_credit; /* accept syscall limit across all addresses */ + long admit_updated; /* when admit_credit was last refilled */ + ulong admit_credit; /* new candidate limit across all addresses */ + long reconnect_updated; + ulong reconnect_credit; + long hello_timeout; /* how long a candidate has to pair */ + long silence_timeout; /* a paired session silent for this long is dropped */ + long last_rx; /* when the paired session last received a frame */ + long keepalive_at; + long retry_at; /* when the dialer dials again */ + long paired_at; /* when the current session paired */ + long backoff; /* the current dial backoff, doubles up to backoff_max */ + long backoff_min; /* the shortest dial backoff, also the jitter range */ + long backoff_max; /* the longest dial backoff */ + long dial_log_at[ 3 ]; /* connect, early close, handshake timeout */ + long tls_log_at[ 4 ]; /* transport, ALPN, authentication, other */ + long hello_log_at[ 14 ]; /* one bounded bucket per HELLO refusal */ + long loss_log_at[ 6 ]; /* transport, silence, TLS, protocol, framing, local encode */ + long pair_log_at; /* when a pairing may be logged again */ + int expected_close; /* RESULT was sent, peer may close normally */ + ulong generation; /* identifies the current authenticated session */ }; FD_FN_CONST ulong fd_failover_channel_align ( void ) { return alignof(fd_failover_channel_t); } @@ -95,7 +153,11 @@ fd_failover_channel_new( void * shmem ) { ch->backoff_min = FD_FAILOVER_CHANNEL_BACKOFF_MIN_NANOS; ch->backoff_max = FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS; ch->backoff = ch->backoff_min; - ch->rate_credit = 16UL*FD_FAILOVER_NANOS_PER_SECOND; + ch->rate_credit = FD_FAILOVER_ACCEPT_BURST*FD_FAILOVER_NANOS_PER_SECOND; + ch->admit_credit = FD_FAILOVER_ADMIT_BURST*FD_FAILOVER_NANOS_PER_SECOND; + ch->verify_credit = FD_FAILOVER_VERIFY_BURST*FD_FAILOVER_NANOS_PER_SECOND; + ch->priority_verify_credit = FD_FAILOVER_PRIORITY_VERIFY_BURST*FD_FAILOVER_NANOS_PER_SECOND; + ch->reconnect_credit = FD_FAILOVER_RECONNECT_BURST*FD_FAILOVER_NANOS_PER_SECOND; FD_TEST( fd_sha512_join( fd_sha512_new( ch->sha ) ) ); ch->magic = FD_FAILOVER_CHANNEL_MAGIC; return shmem; @@ -110,7 +172,8 @@ fd_failover_channel_join( void * shmem ) { static void close_candidate( struct candidate * c ) { - fd_failover_tls_fini( &c->tls ); + if( c->tls_started ) fd_failover_tls_fini( &c->tls ); + c->tls_started = 0; if( c->fd!=-1 ) close( c->fd ); /* Closing a candidate frees its slot. */ c->fd = -1; @@ -128,7 +191,7 @@ reset( fd_failover_channel_t * ch ) { fd_memset( &ch->peer_hello, 0, sizeof(ch->peer_hello) ); } -/* The controller sets a dial address only for an open handoff. */ +/* A standby may authenticate before its operator requests a handoff. */ static int want_dial( fd_failover_channel_t const * ch ) { return !!ch->peer_addr; @@ -158,7 +221,7 @@ fd_failover_channel_init_listener( fd_failover_channel_t * ch, int yes = 1; if( setsockopt( ch->listen_fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes) ) ) FD_LOG_ERR(( "failover setsockopt failed" )); struct sockaddr_in addr = { .sin_family=AF_INET, .sin_port=fd_ushort_bswap( port ), .sin_addr.s_addr=address }; - if( bind( ch->listen_fd, fd_type_pun( &addr ), sizeof(addr) ) || listen( ch->listen_fd, 64 ) ) + if( bind( ch->listen_fd, fd_type_pun( &addr ), sizeof(addr) ) || listen( ch->listen_fd, FD_FAILOVER_LISTEN_BACKLOG ) ) FD_LOG_ERR(( "could not listen on `" FD_IP4_ADDR_FMT ":%hu` from [failover.listen_address] and [failover.listen_port] (%i-%s)", FD_IP4_ADDR_FMT_ARGS( address ), port, errno, fd_io_strerror( errno ) )); } @@ -168,6 +231,7 @@ fd_failover_channel_init_dialer( fd_failover_channel_t * ch, uint address, ushort port ) { if( address==ch->peer_addr && port==ch->peer_port ) return; + if( address && ch->paired_idx>=0 ) reset( ch ); ch->peer_addr = address; ch->peer_port = port; if( address ) { @@ -182,28 +246,20 @@ int fd_failover_channel_set_identity( fd_failover_channel_t * ch, uchar const junk_pubkey[ 32 ], fd_tls_sign_t signer, - fd_failover_hello_t const * hello ) { - if( !fd_memeq( junk_pubkey, hello->junk_pubkey, 32UL ) || + fd_failover_proof_signer_t proof_signer, + fd_failover_hello_t const * hello ) { + if( !proof_signer.sign_fn || !signer.sign_fn || + !fd_memeq( junk_pubkey, hello->junk_pubkey, 32UL ) || fd_memeq( hello->junk_pubkey, hello->staked_pubkey, 32UL ) ) return -1; reset( ch ); /* An identity change requires a fresh session. */ + fd_memset( ch->known_sources, 0, sizeof(ch->known_sources) ); ch->state = fd_failover_session_init( ch->dial_peer ); - ch->self_hello = *hello; - ch->member_cert_set = 0; + ch->self_hello = *hello; + ch->proof_signer = proof_signer; return fd_failover_tls_ctx_init( &ch->tls_ctx, junk_pubkey, signer ); } -int -fd_failover_channel_set_member_cert( fd_failover_channel_t * ch, - uchar const * member_cert ) { - fd_failover_hello_t hello = ch->self_hello; - fd_memcpy( hello.member_cert, member_cert, 64UL ); - if( FD_UNLIKELY( fd_failover_member_cert_check( &hello, ch->sha )!=FD_FAILOVER_HELLO_OK ) ) return -1; - ch->self_hello = hello; - ch->member_cert_set = 1; - return 0; -} - void fd_failover_channel_fini( fd_failover_channel_t * ch ) { reset( ch ); @@ -212,10 +268,17 @@ fd_failover_channel_fini( fd_failover_channel_t * ch ) { fd_failover_tls_ctx_fini( &ch->tls_ctx ); } +fd_failover_channel_metrics_t const * +fd_failover_channel_metrics( fd_failover_channel_t const * ch ) { return &ch->metrics; } + void fd_failover_channel_set_role( fd_failover_channel_t * ch, ulong role ) { + if( ch->self_hello.role==(uchar)role ) return; ch->self_hello.role = (uchar)role; + for( ulong i=0UL; ipaired_idx ) close_candidate( &ch->candidates[i] ); + if( ch->paired_idx<0 ) rest( ch ); } FD_FN_PURE ulong fd_failover_channel_state ( fd_failover_channel_t const * ch ) { return ch->state; } @@ -336,6 +399,7 @@ fd_failover_channel_hangup( fd_failover_channel_t * ch, void fd_failover_channel_protocol_error( fd_failover_channel_t * ch, long now ) { + ch->metrics.wire_fatal_cnt++; if( FD_LIKELY( ch->paired_idx>=0 ) ) { lose( ch, (ulong)ch->paired_idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_PROTOCOL, "it sent a message that breaks the protocol, check that both machines run the same Firedancer version" ); } @@ -357,81 +421,84 @@ refill( ulong credit, return credit; } -static int -admit( fd_failover_channel_t * ch, - uint address, - long now ) { - ch->rate_credit = refill( ch->rate_credit, &ch->rate_updated, now, 32UL, 16UL ); - if( ch->rate_creditcandidates[i].fd!=-1 && ch->candidates[i].address==address; - if( same>=2UL ) return 0; +/* Records are created only for allocated candidates. The admission + budget bounds their count through the longest quota and quiet period. */ +static struct source * +source_get( fd_failover_channel_t * ch, + uint address, + long now, + int create ) { struct source * source = NULL; struct source * reusable = NULL; for( ulong i=0; isources[i]; - if( s->used ) s->credit = refill( s->credit, &s->updated, now, 4UL, 2UL ); if( s->used && s->address==address ) { source = s; break; } - if( !s->used || s->credit==2UL*FD_FAILOVER_NANOS_PER_SECOND ) reusable = s; + if( !reusable || ( reusable->used && ( !s->used || s->last_admitlast_admit ) ) ) reusable = s; } if( !source ) { - if( !reusable ) return 0; + if( !create ) return NULL; source = reusable; - *source = (struct source){ .address=address, .used=1, .updated=now, .credit=2UL*FD_FAILOVER_NANOS_PER_SECOND }; + *source = (struct source){ .address=address, .used=1, .last_admit=now }; } - if( source->creditcredit -= FD_FAILOVER_NANOS_PER_SECOND; - ch->rate_credit -= FD_FAILOVER_NANOS_PER_SECOND; - return 1; + return source; } -/* A full table does not keep our dial out. The oldest candidate from - the listener makes room, so holding every slot only ever costs the - holder a slot. */ -static ulong -dial_slot( fd_failover_channel_t * ch, - long now ) { - ulong victim = FD_FAILOVER_CHANNEL_CANDIDATE_MAX; - for( ulong i=0; icandidates[i]; - if( c->fd==-1 ) return i; - if( (int)i==ch->paired_idx || c->dialed ) continue; - if( victim==FD_FAILOVER_CHANNEL_CANDIDATE_MAX || c->deadlinecandidates[victim].deadline ) victim = i; - } - if( victim==FD_FAILOVER_CHANNEL_CANDIDATE_MAX ) return victim; - drop( ch, victim, now, FD_FAILOVER_EV_TIMEOUT ); - return victim; +static int +admit( fd_failover_channel_t * ch, + uint address, + long now ) { + for( ulong i=RECONNECT_IDX; icandidates[i].fd!=-1 && ch->candidates[i].address==address ) return 0; + struct source const * source = source_get( ch, address, now, 0 ); + return !source || now>=source->next_admit; } -/* A full table admits a newcomer the listener let in by closing an - inbound candidate that has not authenticated. The address holding the - most candidates loses its oldest first, so a flood of connections - mostly displaces itself, and a candidate that alone holds its address - goes only once every address holds one. */ +static void +source_admitted( fd_failover_channel_t * ch, + uint address, + long now ) { + struct source * source = source_get( ch, address, now, 1 ); + if( !source->backoff || now>=fd_long_sat_add( source->next_admit, FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS ) ) + source->backoff = ch->hello_timeout; + else + source->backoff = fd_long_min( fd_long_sat_add( source->backoff, source->backoff ), FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS ); + source->last_admit = now; + source->next_admit = fd_long_sat_add( now, source->backoff ); +} + +static int +proof_ready( struct candidate const * c ) { + return c->phase==PHASE_REQUEST_RECV && c->rx_used==sizeof(c->client_proof); +} + +/* Admission never displaces an existing handshake. Its phase and + absolute deadlines bound retention even when it stops progressing. */ static ulong listen_slot( fd_failover_channel_t * ch, - long now ) { - ulong victim = FD_FAILOVER_CHANNEL_CANDIDATE_MAX; - ulong victim_cnt = 0UL; - for( ulong i=0; icandidates[i]; - if( c->fd==-1 ) return i; - if( (int)i==ch->paired_idx || c->dialed || c->phase==PHASE_READY ) continue; - ulong cnt = 0UL; - for( ulong j=0; jcandidates[j]; - cnt += o->fd!=-1 && !o->dialed && o->address==c->address; - } - if( victim==FD_FAILOVER_CHANNEL_CANDIDATE_MAX || cnt>victim_cnt || - ( cnt==victim_cnt && c->deadlinecandidates[victim].deadline ) ) { - victim = i; - victim_cnt = cnt; - } + long now FD_PARAM_UNUSED ) { + for( ulong i=UNAUTHENTICATED_IDX; icandidates[i].fd==-1 ) return i; } - if( victim==FD_FAILOVER_CHANNEL_CANDIDATE_MAX ) return victim; - drop( ch, victim, now, FD_FAILOVER_EV_TIMEOUT ); - return victim; + return FD_FAILOVER_CHANNEL_CANDIDATE_MAX; +} + +static int +known_source( fd_failover_channel_t const * ch, uint address, long now ) { + for( ulong i=0UL; iknown_sources[i].address==address && nowknown_sources[i].until ) return 1; + return 0; +} + +/* Only a completed mutual authentication or a frame on that session + refreshes this cache. An IP grants scheduling priority, not trust. */ +static void +remember_source( fd_failover_channel_t * ch, uint address, long now ) { + ulong idx = 0UL; + for( ulong i=0UL; iknown_sources[i].address==address ) { idx = i; break; } + if( ch->known_sources[i].untilknown_sources[idx].until ) idx = i; + } + ch->known_sources[idx] = (struct known_source){ .address=address, .until=fd_long_sat_add( now, ch->silence_timeout ) }; } static int @@ -442,14 +509,16 @@ start( fd_failover_channel_t * ch, long now, int phase ) { struct candidate * c = &ch->candidates[idx]; - /* Outbound sockets start at CONNECT, accepted sockets start at TLS. */ + /* Inbound sockets do not initialize TLS until admission succeeds. */ c->fd = fd; c->address = address; c->phase = phase; c->dialed = phase==PHASE_CONNECT; - c->deadline = fd_long_sat_add( now, ch->hello_timeout ); + c->priority = c->dialed || idx==RECONNECT_IDX; + c->deadline = fd_long_sat_add( now, ch->hello_timeout ); + c->admission_until = c->dialed ? c->deadline : fd_long_min( c->deadline, fd_long_sat_add( now, FD_FAILOVER_ADMISSION_START_NANOS ) ); + c->accepted_at = now; + c->local_hello = ch->self_hello; + c->rx_used = c->tx_used = c->tx_sent = 0UL; + c->tls_started = 0; fd_failover_wire_session_init( &c->wire ); - if( fd_failover_tls_new( &c->tls, &ch->tls_ctx, fd, c->dialed ) ) { - drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); - return -1; - } return 0; } @@ -457,24 +526,48 @@ static void accept_candidates( fd_failover_channel_t * ch, long now, int * busy ) { - for( ulong n=0; n<8UL; n++ ) { + for( ulong n=0; nrate_credit = refill( ch->rate_credit, &ch->rate_updated, now, FD_FAILOVER_ACCEPT_RATE, FD_FAILOVER_ACCEPT_BURST ); + if( ch->rate_creditmetrics.accept_limited_cnt++; return; } + ch->rate_credit -= FD_FAILOVER_NANOS_PER_SECOND; struct sockaddr_in addr; socklen_t len = sizeof(addr); int fd = accept4( ch->listen_fd, fd_type_pun( &addr ), &len, SOCK_NONBLOCK|SOCK_CLOEXEC ); if( fd==-1 ) return; + ch->metrics.accept_cnt++; *busy = 1; - if( ch->paired_idx>=0 || len!=sizeof(addr) || addr.sin_family!=AF_INET || !admit( ch, addr.sin_addr.s_addr, now ) ) { + ch->admit_credit = refill( ch->admit_credit, &ch->admit_updated, now, FD_FAILOVER_ADMIT_RATE, FD_FAILOVER_ADMIT_BURST ); + ch->reconnect_credit = refill( ch->reconnect_credit, &ch->reconnect_updated, now, FD_FAILOVER_RECONNECT_RATE, FD_FAILOVER_RECONNECT_BURST ); + if( ch->paired_idx>=0 || len!=sizeof(addr) || addr.sin_family!=AF_INET ) { + close( fd ); + continue; + } + int priority = known_source( ch, addr.sin_addr.s_addr, now ); + ulong * credit = priority ? &ch->reconnect_credit : &ch->admit_credit; + if( *creditmetrics.budget_reject_cnt++; close( fd ); continue; } - ulong idx = listen_slot( ch, now ); + ulong idx = priority ? RECONNECT_IDX : listen_slot( ch, now ); + if( priority && ch->candidates[idx].fd!=-1 ) idx = FD_FAILOVER_CHANNEL_CANDIDATE_MAX; if( FD_UNLIKELY( idx==FD_FAILOVER_CHANNEL_CANDIDATE_MAX ) ) { + ch->metrics.capacity_reject_cnt++; + close( fd ); + continue; + } + if( !admit( ch, addr.sin_addr.s_addr, now ) ) { + ch->metrics.source_reject_cnt++; close( fd ); continue; } /* An accepted candidate leaves the session unpaired until authentication. */ - if( !start( ch, idx, fd, addr.sin_addr.s_addr, now, PHASE_TLS ) ) + if( !start( ch, idx, fd, addr.sin_addr.s_addr, now, PHASE_START_RECV ) ) { + *credit -= FD_FAILOVER_NANOS_PER_SECOND; + ch->metrics.reconnect_cnt += (ulong)priority; + source_admitted( ch, addr.sin_addr.s_addr, now ); ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_PEER_CONNECTED ); + } } } @@ -485,7 +578,7 @@ flush( fd_failover_channel_t * ch, int * busy ) { struct candidate * c = &ch->candidates[idx]; if( !c->tx_used ) { - int rc = fd_tlsrec_sock_flush( &c->tls.sock, c->fd ); + int rc = fd_failover_tls_flush( &c->tls ); if( rc<0 ) { lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_TRANSPORT, "flushing TLS output failed" ); return -1; } return rc; } @@ -534,11 +627,22 @@ hello_err_name( int err ) { case FD_FAILOVER_HELLO_ERR_ROLE: return "the role is unknown, run the same Firedancer version on both machines"; case FD_FAILOVER_HELLO_ERR_BOOT_ID: return "the boot id is zero, run the same Firedancer version on both machines"; case FD_FAILOVER_HELLO_ERR_MODE: return "the consensus mode differs, run both machines in the same consensus mode"; - case FD_FAILOVER_HELLO_ERR_CERT: return "its member certificate is not signed by the failover identity, check that both machines hold the same keypair for it"; - default: return "unknown"; + case FD_FAILOVER_HELLO_ERR_PROOF: return "its admission proof is not signed by the failover identity, check that both machines hold the same keypair for it"; + default: return "unknown"; } } +static void +hello_refused( fd_failover_channel_t * ch, uint address, int err, long now ) { + ch->metrics.hello_reject_cnt++; + ch->metrics.proof_reject_cnt += (ulong)( err==FD_FAILOVER_HELLO_ERR_PROOF ); + ulong bucket = err==HELLO_ERR_TLS_KEY ? 12UL : ( err>0 && err<=FD_FAILOVER_HELLO_ERR_PROOF ? (ulong)err : 13UL ); + if( nowhello_log_at[bucket] ) return; + ch->hello_log_at[bucket] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); + FD_LOG_WARNING(( "rejected the failover peer at `" FD_IP4_ADDR_FMT "`, %s", + FD_IP4_ADDR_FMT_ARGS( address ), hello_err_name( err ) )); +} + static int read_frame( fd_failover_channel_t * ch, ulong idx, @@ -597,8 +701,28 @@ connect_candidate( fd_failover_channel_t * ch, drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); return 0; } - /* TCP connected, begin TLS and enter the session HELLO state. */ - c->phase = PHASE_TLS; + /* Only operator-initiated outbound work prepares a TLS key share here. */ + if( fd_failover_tls_new( &c->tls, &ch->tls_ctx, c->fd, 1 ) ) { + drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); + return 0; + } + c->tls_started = 1; + ulong hello_sz = sizeof(c->client_hello); + if( fd_failover_tls_client_hello( &c->tls, c->client_hello, &hello_sz ) || + hello_sz>sizeof(c->client_hello) ) { + drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); + return 0; + } + c->start = (fd_failover_admission_start_t){ .magic=FD_FAILOVER_ADMISSION_MAGIC, + .hello=c->local_hello, .client_hello_sz=(ushort)hello_sz }; + if( !fd_rng_secure( c->start.nonce, sizeof(c->start.nonce) ) ) { + drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); + return 0; + } + fd_memcpy( c->tx, &c->start, sizeof(c->start) ); + fd_memcpy( c->tx+sizeof(c->start), c->client_hello, hello_sz ); + c->tx_used = sizeof(c->start)+hello_sz; + c->phase = PHASE_START_SEND; ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_CONNECTED ); return 1; } @@ -611,6 +735,7 @@ handshake_candidate( fd_failover_channel_t * ch, struct candidate * c = &ch->candidates[idx]; int rc = fd_failover_tls_handshake( &c->tls ); if( rc<0 ) { + ch->metrics.tls_fail_cnt++; uint reason = c->tls.conn.hs.base.reason; ulong kind = tls_log_kind( reason ); if( now>=ch->tls_log_at[kind] ) { @@ -629,7 +754,7 @@ handshake_candidate( fd_failover_channel_t * ch, /* The dialer sends first. On the listener we respond only to a HELLO that passed the checks below. */ if( c->dialed ) c->tx_used = fd_failover_wire_encode( &c->wire, c->tx, FD_FAILOVER_MSG_HELLO, - (uchar const *)&ch->self_hello, sizeof(ch->self_hello) ); + (uchar const *)&c->local_hello, sizeof(c->local_hello) ); return 1; } @@ -652,17 +777,12 @@ read_candidate_hello( fd_failover_channel_t * ch, drop( ch, idx, now, FD_FAILOVER_EV_HELLO_FATAL ); return 0; } - fd_memcpy( &c->hello, payload, sizeof(c->hello) ); - /* The junk key in HELLO has to be the key TLS authenticated, and the - member certificate proves the peer holds the staked key. */ - int err = fd_memeq( c->hello.junk_pubkey, c->tls.peer_pubkey, 32UL ) ? fd_failover_hello_check( &ch->self_hello, &c->hello ) : HELLO_ERR_TLS_KEY; - if( err==FD_FAILOVER_HELLO_OK ) err = fd_failover_member_cert_check( &c->hello, ch->sha ); + /* HELLO and TLS must match the admission transcript exactly. */ + int err = fd_memeq( payload, &c->hello, sizeof(c->hello) ) && + fd_memeq( c->hello.junk_pubkey, c->tls.peer_pubkey, 32UL ) + ? fd_failover_hello_check( &c->local_hello, &c->hello ) : HELLO_ERR_TLS_KEY; if( err!=FD_FAILOVER_HELLO_OK ) { - ulong bucket = err==HELLO_ERR_TLS_KEY ? 12UL : ( err>0 && err<=FD_FAILOVER_HELLO_ERR_CERT ? (ulong)err : 13UL ); - if( now>=ch->hello_log_at[bucket] ) { - ch->hello_log_at[bucket] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); - FD_LOG_WARNING(( "rejected the failover HELLO from `" FD_IP4_ADDR_FMT "`, %s", FD_IP4_ADDR_FMT_ARGS( c->address ), hello_err_name( err ) )); - } + hello_refused( ch, c->address, err, now ); /* A refused HELLO closes this candidate without pairing. */ drop( ch, idx, now, FD_FAILOVER_EV_HELLO_FATAL ); return 0; @@ -671,7 +791,7 @@ read_candidate_hello( fd_failover_channel_t * ch, c->phase = PHASE_READY; if( !c->dialed ) { c->tx_used = fd_failover_wire_encode( &c->wire, c->tx, FD_FAILOVER_MSG_HELLO, - (uchar const *)&ch->self_hello, sizeof(ch->self_hello) ); + (uchar const *)&c->local_hello, sizeof(c->local_hello) ); if( flush( ch, idx, now, busy )!=0 ) return 0; /* Remain READY until output drains, unless the write fails. */ } return 1; @@ -697,6 +817,11 @@ pair_candidate( fd_failover_channel_t * ch, /* HELLO is complete, this candidate becomes the paired session. */ ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_HELLO_OK ); ch->generation++; + remember_source( ch, c->address, now ); + ch->metrics.paired_cnt++; + ch->metrics.handshake_latest_nanos = (ulong)fd_long_max( now-c->accepted_at, 0L ); + ch->metrics.handshake_duration_nanos += ch->metrics.handshake_latest_nanos; + ch->keepalive_at = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_KEEPALIVE_NANOS ); ch->last_rx = now; ch->paired_at = now; ch->expected_close = 0; @@ -711,6 +836,229 @@ pair_candidate( fd_failover_channel_t * ch, } } +/* Each admission service turn performs at most one socket operation. + No input can extend the absolute candidate deadline. */ +static int +admission_read( struct candidate * c, ulong goal, int * busy ) { + if( FD_UNLIKELY( goal>FD_FAILOVER_ADMISSION_MAX || c->rx_used>goal ) ) return -1; + if( c->rx_used==goal ) return 1; + long n = recv( c->fd, c->rx+c->rx_used, goal-c->rx_used, MSG_DONTWAIT|MSG_NOSIGNAL ); + if( n<0L ) return errno==EAGAIN || errno==EWOULDBLOCK || errno==EINTR ? 0 : -1; + if( !n ) return -1; + c->rx_used += (ulong)n; + *busy = 1; + return c->rx_used==goal; +} + +static int +admission_write( struct candidate * c, int * busy ) { + long n = send( c->fd, c->tx+c->tx_sent, c->tx_used-c->tx_sent, MSG_DONTWAIT|MSG_NOSIGNAL ); + if( n<0L ) return errno==EAGAIN || errno==EWOULDBLOCK || errno==EINTR ? 0 : -1; + if( !n ) return -1; + c->tx_sent += (ulong)n; + *busy = 1; + if( c->tx_senttx_used ) return 0; + c->tx_sent = c->tx_used = 0UL; + return 1; +} + +static int +proof_verify( fd_failover_channel_t * ch, struct candidate * c, int server, uchar const sig[ 64 ] ) { + uchar msg[ FD_KEYGUARD_FAILOVER_PROOF_SZ ]; + fd_failover_proof_msg( msg, server, c->hello.junk_pubkey, c->transcript ); + return fd_ed25519_verify( msg, sizeof(msg), sig, c->local_hello.staked_pubkey, ch->sha )==FD_ED25519_SUCCESS; +} + +static int +proof_sign( fd_failover_channel_t * ch, struct candidate * c, int server ) { + uchar msg[ FD_KEYGUARD_FAILOVER_PROOF_SZ ]; + fd_failover_proof_msg( msg, server, c->local_hello.junk_pubkey, c->transcript ); + ch->proof_signer.sign_fn( ch->proof_signer.ctx, c->tx, msg ); + /* An operator identity change may have reached Sign first. */ + if( fd_ed25519_verify( msg, sizeof(msg), c->tx, c->local_hello.staked_pubkey, ch->sha )!=FD_ED25519_SUCCESS ) return 0; + c->tx_used = 64UL; + c->tx_sent = 0UL; + return 1; +} + +static int +verify_credit( fd_failover_channel_t * ch, struct candidate const * c, long now ) { + ch->verify_credit = refill( ch->verify_credit, &ch->verify_updated, now, FD_FAILOVER_VERIFY_RATE, FD_FAILOVER_VERIFY_BURST ); + ch->priority_verify_credit = refill( ch->priority_verify_credit, &ch->priority_verify_updated, now, + FD_FAILOVER_PRIORITY_VERIFY_RATE, FD_FAILOVER_PRIORITY_VERIFY_BURST ); + ulong * credit = c->priority ? &ch->priority_verify_credit : &ch->verify_credit; + if( !ch->verify_budget || *creditverify_budget--; + *credit -= FD_FAILOVER_NANOS_PER_SECOND; + return 1; +} + +static void +admission_service( fd_failover_channel_t * ch, ulong idx, long now, int * busy ) { + struct candidate * c = &ch->candidates[idx]; + int rc; + switch( c->phase ) { + case PHASE_START_SEND: + case PHASE_CHALLENGE_SEND: + case PHASE_REQUEST_SEND: + case PHASE_RESPONSE_SEND: + rc = admission_write( c, busy ); + if( rc<0 ) break; + if( !rc ) return; + c->rx_used = 0UL; + if( c->phase==PHASE_START_SEND ) { + c->phase = PHASE_CHALLENGE_RECV; + } + else if( c->phase==PHASE_CHALLENGE_SEND ) { + c->admission_until = fd_long_min( c->deadline, fd_long_sat_add( now, FD_FAILOVER_ADMISSION_PROOF_NANOS ) ); + c->phase = PHASE_REQUEST_RECV; + } + else if( c->phase==PHASE_REQUEST_SEND ) { + c->phase = PHASE_RESPONSE_RECV; + } + else { + c->phase = PHASE_TLS; + } + return; + case PHASE_START_RECV: { + /* Read only the fixed header until its length is checked. */ + ulong goal = c->rx_usedstart) ? sizeof(c->start) : sizeof(c->start)+c->start.client_hello_sz; + rc = admission_read( c, goal, busy ); + if( rc<0 ) break; + if( !rc ) return; + if( goal==sizeof(c->start) ) { + fd_memcpy( &c->start, c->rx, sizeof(c->start) ); + if( c->start.magic!=FD_FAILOVER_ADMISSION_MAGIC || + c->start.client_hello_sz<=sizeof(fd_tlsrec_hdr_t) || + c->start.client_hello_sz>sizeof(c->client_hello) ) break; + int err = fd_failover_hello_check( &c->local_hello, &c->start.hello ); + if( err ) { hello_refused( ch, c->address, err, now ); break; } + return; + } + fd_memcpy( c->client_hello, c->rx+sizeof(c->start), c->start.client_hello_sz ); + fd_tlsrec_hdr_t hdr; + fd_memcpy( &hdr, c->client_hello, sizeof(hdr) ); + if( hdr.content_type!=FD_TLS_REC_HANDSHAKE || + (ulong)fd_ushort_bswap( hdr.length )+sizeof(hdr)!=c->start.client_hello_sz || + c->client_hello[ sizeof(hdr) ]!=FD_TLS_MSG_CLIENT_HELLO ) break; + c->hello = c->start.hello; + c->challenge = (fd_failover_admission_challenge_t){ .magic=FD_FAILOVER_ADMISSION_MAGIC, .hello=c->local_hello }; + if( !fd_rng_secure( c->challenge.nonce, sizeof(c->challenge.nonce) ) ) break; + fd_failover_admission_hash( &c->start, c->client_hello, &c->challenge, c->transcript ); + fd_memcpy( c->tx, &c->challenge, sizeof(c->challenge) ); + c->tx_used = sizeof(c->challenge); + c->rx_used = 0UL; + c->admission_until = fd_long_min( c->deadline, fd_long_sat_add( now, FD_FAILOVER_ADMISSION_START_NANOS ) ); + c->phase = PHASE_CHALLENGE_SEND; + return; + } + case PHASE_CHALLENGE_RECV: + rc = admission_read( c, sizeof(c->challenge), busy ); + if( rc<0 ) break; + if( !rc ) return; + fd_memcpy( &c->challenge, c->rx, sizeof(c->challenge) ); + if( c->challenge.magic!=FD_FAILOVER_ADMISSION_MAGIC ) break; + int err = fd_failover_hello_check( &c->local_hello, &c->challenge.hello ); + if( err ) { hello_refused( ch, c->address, err, now ); break; } + c->hello = c->challenge.hello; + fd_failover_admission_hash( &c->start, c->client_hello, &c->challenge, c->transcript ); + c->rx_used = 0UL; + c->phase = PHASE_REQUEST_SIGN; + *busy = 1; + return; + case PHASE_REQUEST_SIGN: + if( !verify_credit( ch, c, now ) ) return; + if( !proof_sign( ch, c, 0 ) ) break; + fd_memcpy( c->client_proof, c->tx, sizeof(c->client_proof) ); + c->phase = PHASE_REQUEST_SEND; + *busy = 1; + return; + case PHASE_RESPONSE_RECV: { + fd_failover_admission_response_t response; + if( c->rx_usedrx, sizeof(response) ); + if( !response.server_flight_sz || response.server_flight_sz>FD_FAILOVER_SERVER_FLIGHT_MAX ) break; + return; + } + fd_memcpy( &response, c->rx, sizeof(response) ); + rc = admission_read( c, sizeof(response)+response.server_flight_sz, busy ); + if( rc<0 ) break; + if( !rc || !verify_credit( ch, c, now ) ) return; + fd_failover_response_hash( c->transcript, c->client_proof, c->rx+sizeof(response), + response.server_flight_sz, c->transcript ); + if( !proof_verify( ch, c, 1, response.proof ) ) { + hello_refused( ch, c->address, FD_FAILOVER_HELLO_ERR_PROOF, now ); + break; + } + if( fd_failover_tls_accept_flight( &c->tls, c->rx+sizeof(response), response.server_flight_sz ) ) break; + c->phase = PHASE_TLS; + c->rx_used = 0UL; + *busy = 1; + return; + } + case PHASE_REQUEST_RECV: + rc = admission_read( c, 64UL, busy ); + if( rc<0 ) break; + if( !rc || !verify_credit( ch, c, now ) ) return; + if( !proof_verify( ch, c, 0, c->rx ) ) { + hello_refused( ch, c->address, FD_FAILOVER_HELLO_ERR_PROOF, now ); + break; + } + struct source * source = source_get( ch, c->address, now, 0 ); + if( source ) { + source->backoff = 0L; + source->next_admit = 0L; + } + if( ch->candidates[ ADMITTED_IDX ].fd!=-1 ) break; + fd_memcpy( c->client_proof, c->rx, sizeof(c->client_proof) ); + c->rx_used = 0UL; + c->admission_until = c->deadline; + c->phase = PHASE_TLS_START; + c->priority = 1; + /* Only a verified proof can claim this protected slot. */ + ch->candidates[ ADMITTED_IDX ] = *c; + c->fd = -1; + c->tls_started = 0; + *busy = 1; + return; + case PHASE_RESPONSE_SIGN: { + if( !verify_credit( ch, c, now ) ) return; + fd_failover_admission_response_t response = { .server_flight_sz=(ushort)(c->tx_used-sizeof(fd_failover_admission_response_t)) }; + if( !proof_sign( ch, c, 1 ) ) break; + fd_memcpy( response.proof, c->tx, sizeof(response.proof) ); + fd_memcpy( c->tx, &response, sizeof(response) ); + c->tx_used = sizeof(response)+response.server_flight_sz; + c->phase = PHASE_RESPONSE_SEND; + *busy = 1; + return; + } + case PHASE_TLS_START: + if( fd_failover_tls_new( &c->tls, &ch->tls_ctx, c->fd, 0 ) ) break; + c->tls_started = 1; + if( fd_failover_tls_accept_flight( &c->tls, c->client_hello, c->start.client_hello_sz ) ) break; + if( !c->tls.sock.tx_sz || c->tls.sock.tx_sz>FD_FAILOVER_SERVER_FLIGHT_MAX ) break; + fd_failover_response_hash( c->transcript, c->client_proof, c->tls.sock.tx_buf, + (ushort)c->tls.sock.tx_sz, c->transcript ); + fd_memcpy( c->tx+sizeof(fd_failover_admission_response_t), c->tls.sock.tx_buf, c->tls.sock.tx_sz ); + c->tx_used = sizeof(fd_failover_admission_response_t)+c->tls.sock.tx_sz; + c->tls.sock.tx_off = c->tls.sock.tx_sz = 0UL; + c->phase = PHASE_RESPONSE_SIGN; + *busy = 1; + return; + default: + break; + } + if( c->dialed && now>=ch->dial_log_at[1] ) { + ch->dial_log_at[1] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); + FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` did not complete admission, check its log for an identity or vote account mismatch and that both machines run the same version", + FD_IP4_ADDR_FMT_ARGS( c->address ) )); + } + drop( ch, idx, now, FD_FAILOVER_EV_HELLO_FATAL ); +} + static void service_candidate( fd_failover_channel_t * ch, ulong idx, @@ -720,7 +1068,12 @@ service_candidate( fd_failover_channel_t * ch, struct candidate * c = &ch->candidates[idx]; fd_failover_tls_budget( &c->tls ); if( c->phase==PHASE_CONNECT ) { - if( !connect_candidate( ch, idx, now ) ) return; + if( connect_candidate( ch, idx, now ) ) *busy = 1; + return; + } + if( c->phasephase==PHASE_TLS ) { if( !handshake_candidate( ch, idx, now, busy ) ) return; @@ -742,6 +1095,8 @@ fd_failover_channel_poll( fd_failover_channel_t * ch, uchar * payload, ulong * payload_sz ) { int delivered = 0; + ch->verify_budget = 1; + ch->verify_credit = refill( ch->verify_credit, &ch->verify_updated, now, FD_FAILOVER_VERIFY_RATE, FD_FAILOVER_VERIFY_BURST ); /* Established traffic gets service before the bounded accept drain. */ if( ch->paired_idx>=0 ) { ulong idx = (ulong)ch->paired_idx; @@ -751,7 +1106,11 @@ fd_failover_channel_poll( fd_failover_channel_t * ch, int rc = read_frame( ch, idx, now, busy, type, payload, payload_sz ); if( rc>0 ) { if( *type==FD_FAILOVER_MSG_HELLO ) { lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_PROTOCOL, "it sent a second HELLO, check that both machines run the same Firedancer version" ); } - else { ch->last_rx = now; delivered = 1; } + else { + ch->last_rx = now; + remember_source( ch, ch->candidates[idx].address, now ); + delivered = *type!=FD_FAILOVER_MSG_KEEPALIVE; + } } else if( !rc && now>=ch->last_rx && now-ch->last_rx>ch->silence_timeout ) { lose( ch, idx, now, FD_FAILOVER_EV_TIMEOUT, LOSS_SILENCE, "the handoff connection was idle past its deadline, check the peer log" ); } @@ -759,45 +1118,73 @@ fd_failover_channel_poll( fd_failover_channel_t * ch, if( flushed>0 && now>=ch->last_rx && now-ch->last_rx>ch->silence_timeout ) { lose( ch, idx, now, FD_FAILOVER_EV_TIMEOUT, LOSS_SILENCE, "the handoff connection stopped making progress, check the peer log" ); } + if( ch->paired_idx>=0 && !fd_failover_channel_tx_pending( ch ) && now>=ch->keepalive_at ) { + if( !fd_failover_channel_send( ch, now, FD_FAILOVER_MSG_KEEPALIVE, NULL, 0UL ) ) { + ch->keepalive_at = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_KEEPALIVE_NANOS ); + *busy = 1; + } + } } /* Expire every slot each poll, independently of round-robin service. */ for( ulong i=0; icandidates[i]; - if( c->fd!=-1 && (int)i!=ch->paired_idx && now>=c->deadline ) { + if( c->fd!=-1 && (int)i!=ch->paired_idx && ( now>=c->deadline || now>=c->admission_until ) ) { if( c->dialed && now>=ch->dial_log_at[2] ) { ch->dial_log_at[2] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` did not finish the handshake in time, check that it runs and that the machines can reach each other on [failover.listen_port]", FD_IP4_ADDR_FMT_ARGS( c->address ) )); } + ch->metrics.handshake_timeout_cnt++; /* An expired candidate closes, an outbound dial returns to backoff. */ drop( ch, i, now, FD_FAILOVER_EV_TIMEOUT ); *busy = 1; } } - /* Nothing starts before we have our member certificate. Every member - drains accepts, and a member with an open handoff also dials once - the backoff ends. */ - if( FD_UNLIKELY( !ch->member_cert_set ) ) return delivered; - if( ch->listen_fd!=-1 ) accept_candidates( ch, now, busy ); - if( ch->dial_peer && ch->state==FD_FAILOVER_SESSION_BACKOFF && now>=ch->retry_at && ch->tls_ctx.ready && ch->peer_addr ) { - ulong idx = dial_slot( ch, now ); - if( FD_UNLIKELY( idx==FD_FAILOVER_CHANNEL_CANDIDATE_MAX ) ) return delivered; - int fd = socket( AF_INET, SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC, 0 ); - *busy = 1; - if( fd==-1 ) { drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); return delivered; } - /* Backoff expired and an address is available, start dialing. */ - ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_RETRY ); - start( ch, idx, fd, ch->peer_addr, now, PHASE_CONNECT ); + if( FD_UNLIKELY( !ch->tls_ctx.ready ) || ch->paired_idx>=0 ) return delivered; + if( ch->dial_peer && ch->state==FD_FAILOVER_SESSION_BACKOFF && now>=ch->retry_at && ch->peer_addr ) { + ulong idx = OUTBOUND_IDX; + if( ch->candidates[idx].fd==-1 ) { + int fd = socket( AF_INET, SOCK_STREAM|SOCK_NONBLOCK|SOCK_CLOEXEC, 0 ); + *busy = 1; + if( fd==-1 ) { drop( ch, idx, now, FD_FAILOVER_EV_LINK_LOST ); return delivered; } + ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_RETRY ); + start( ch, idx, fd, ch->peer_addr, now, PHASE_CONNECT ); + } } - if( ch->paired_idx>=0 ) return delivered; ulong serviced = 0UL; - for( ulong n=0; ncursor; - ch->cursor = (ch->cursor+1UL)%FD_FAILOVER_CHANNEL_CANDIDATE_MAX; + /* Reserved connections cannot be displaced or wait behind newcomers. */ + for( ulong idx=0UL; idxcandidates[idx].fd==-1 ) continue; service_candidate( ch, idx, now, busy, payload ); serviced++; - if( ch->paired_idx>=0 ) break; + if( ch->paired_idx>=0 ) return delivered; + } + /* Give a buffered proof its verification turn before servicing more + input. An unchecked proof stays in the unauthenticated pool. */ + ulong serviced_mask = 0UL; + if( ch->verify_budget && ch->verify_credit>=FD_FAILOVER_NANOS_PER_SECOND ) { + for( ulong n=0UL; nproof_cursor+n)%(FD_FAILOVER_CHANNEL_CANDIDATE_MAX-UNAUTHENTICATED_IDX); + ulong idx = UNAUTHENTICATED_IDX+cursor; + if( ch->candidates[idx].fd==-1 || !proof_ready( &ch->candidates[idx] ) ) continue; + ch->proof_cursor = (cursor+1UL)%(FD_FAILOVER_CHANNEL_CANDIDATE_MAX-UNAUTHENTICATED_IDX); + service_candidate( ch, idx, now, busy, payload ); + serviced_mask = 1UL<paired_idx>=0 ) return delivered; + break; + } } + for( ulong n=0UL; ncursor; + ch->cursor = (ch->cursor+1UL)%(FD_FAILOVER_CHANNEL_CANDIDATE_MAX-UNAUTHENTICATED_IDX); + if( ch->candidates[idx].fd==-1 || ( serviced_mask & (1UL<candidates[idx] ) && + ( !ch->verify_budget || ch->verify_creditpaired_idx>=0 ) return delivered; + } + if( ch->listen_fd!=-1 ) accept_candidates( ch, now, busy ); return delivered; } diff --git a/src/discof/failover/fd_failover_channel.h b/src/discof/failover/fd_failover_channel.h index 704699e4552..e449de3a873 100644 --- a/src/discof/failover/fd_failover_channel.h +++ b/src/discof/failover/fd_failover_channel.h @@ -9,11 +9,32 @@ #include #define FD_FAILOVER_CHANNEL_MAGIC (0xF17EDA2CE5FA170AUL) -#define FD_FAILOVER_CHANNEL_CANDIDATE_MAX (16UL) +#define FD_FAILOVER_CHANNEL_CANDIDATE_MAX (64UL) #define FD_FAILOVER_NANOS_PER_SECOND (1000000000UL) #define FD_FAILOVER_LOG_INTERVAL_NANOS (60L*(long)FD_FAILOVER_NANOS_PER_SECOND) +#define FD_FAILOVER_ACCEPT_RATE (8192UL) +#define FD_FAILOVER_ACCEPT_BURST (128UL) +#define FD_FAILOVER_ADMIT_RATE (128UL) +#define FD_FAILOVER_ADMIT_BURST (16UL) +#define FD_FAILOVER_VERIFY_RATE (128UL) +#define FD_FAILOVER_VERIFY_BURST (16UL) +#define FD_FAILOVER_ACCEPT_PER_POLL (16UL) +#define FD_FAILOVER_LISTEN_BACKLOG (256) +#define FD_FAILOVER_RECONNECT_RATE (4UL) +#define FD_FAILOVER_RECONNECT_BURST (2UL) +#define FD_FAILOVER_PRIORITY_VERIFY_RATE (16UL) +#define FD_FAILOVER_PRIORITY_VERIFY_BURST (4UL) +#define FD_FAILOVER_ADMISSION_START_NANOS ((long)FD_FAILOVER_NANOS_PER_SECOND) +#define FD_FAILOVER_ADMISSION_PROOF_NANOS (2L*(long)FD_FAILOVER_NANOS_PER_SECOND) + +struct fd_failover_proof_signer { + void * ctx; + void (* sign_fn)( void * ctx, uchar sig[ 64 ], uchar const msg[ FD_KEYGUARD_FAILOVER_PROOF_SZ ] ); +}; +typedef struct fd_failover_proof_signer fd_failover_proof_signer_t; + /* fd_failover_clock returns monotonic nanoseconds for every channel deadline. */ static inline long @@ -27,7 +48,26 @@ fd_failover_clock( void ) { #define FD_FAILOVER_CHANNEL_IDLE_NANOS ( 64000000000L) #define FD_FAILOVER_CHANNEL_BACKOFF_MIN_NANOS ( 800000000L) #define FD_FAILOVER_CHANNEL_BACKOFF_MAX_NANOS ( 12800000000L) -#define FD_FAILOVER_CHANNEL_HELLO_TIMEOUT_NANOS ( 2000000000L) +#define FD_FAILOVER_CHANNEL_HELLO_TIMEOUT_NANOS ( 4000000000L) +#define FD_FAILOVER_CHANNEL_KEEPALIVE_NANOS (FD_FAILOVER_CHANNEL_IDLE_NANOS/4L) + +struct fd_failover_channel_metrics { + ulong accept_cnt; + ulong accept_limited_cnt; + ulong source_reject_cnt; + ulong budget_reject_cnt; + ulong capacity_reject_cnt; + ulong reconnect_cnt; + ulong handshake_timeout_cnt; + ulong proof_reject_cnt; + ulong hello_reject_cnt; + ulong tls_fail_cnt; + ulong wire_fatal_cnt; + ulong paired_cnt; + ulong handshake_duration_nanos; + ulong handshake_latest_nanos; +}; +typedef struct fd_failover_channel_metrics fd_failover_channel_metrics_t; struct fd_failover_channel; typedef struct fd_failover_channel fd_failover_channel_t; @@ -52,29 +92,21 @@ fd_failover_channel_init_listener( fd_failover_channel_t * channel, uint address, ushort port ); -/* Starts a handoff's dial and retries at address:port. */ +/* Starts dialing and retrying at address:port. A changed nonzero + destination replaces an established session. Zero stops dialing. */ void fd_failover_channel_init_dialer( fd_failover_channel_t * channel, uint address, ushort port ); -/* Configures mutual TLS and HELLO with our junk pubkey and the signer - for CertificateVerify. We call it at boot and again after a - set-identity. The private key stays in the sign tile. Returns zero - on success, -1 on invalid keys or TLS setup failure. */ +/* Configures admission and mutual TLS with separate proof and TLS + signers. Identity private keys stay in the sign tile. */ int fd_failover_channel_set_identity( fd_failover_channel_t * channel, uchar const junk_pubkey[ 32 ], fd_tls_sign_t signer, - fd_failover_hello_t const * hello ); - -/* Puts our member certificate, the staked key's signature over the - cert prefix and our junk pubkey, into HELLO. We neither accept nor - dial before we have it. Returns zero on success, -1 if it does not - verify against our HELLO. */ -int -fd_failover_channel_set_member_cert( fd_failover_channel_t * channel, - uchar const * member_cert ); + fd_failover_proof_signer_t proof_signer, + fd_failover_hello_t const * hello ); /* Closes all sockets and releases the TLS context. */ void @@ -120,6 +152,9 @@ FD_FN_PURE int fd_failover_channel_listen_fd ( fd_failov FD_FN_PURE int fd_failover_channel_tx_pending( fd_failover_channel_t const * channel ); FD_FN_PURE fd_failover_hello_t const * fd_failover_channel_peer_hello( fd_failover_channel_t const * channel ); +FD_FN_PURE fd_failover_channel_metrics_t const * +fd_failover_channel_metrics( fd_failover_channel_t const * channel ); + /* Changes whenever a new authenticated session replaces the previous one. */ FD_FN_PURE ulong fd_failover_channel_generation( fd_failover_channel_t const * channel ); diff --git a/src/discof/failover/fd_failover_proto.c b/src/discof/failover/fd_failover_proto.c index 0a1f90ec375..e140a576cb4 100644 --- a/src/discof/failover/fd_failover_proto.c +++ b/src/discof/failover/fd_failover_proto.c @@ -3,9 +3,11 @@ #include "../../choreo/tower/fd_tower.h" #include "../../choreo/tower/fd_tower_serdes.h" #include "../../ballet/ed25519/fd_ed25519.h" +#include "../../ballet/sha256/fd_sha256.h" #include "../../disco/keyguard/fd_keyguard.h" -FD_STATIC_ASSERT( FD_KEYGUARD_MEMBER_CERT_MSG_SZ==FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ+32UL, member_cert_msg ); +FD_STATIC_ASSERT( sizeof(FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN)-1UL==FD_KEYGUARD_FAILOVER_DOMAIN_SZ, client_proof_domain ); +FD_STATIC_ASSERT( sizeof(FD_KEYGUARD_FAILOVER_SERVER_DOMAIN)-1UL==FD_KEYGUARD_FAILOVER_DOMAIN_SZ, server_proof_domain ); int fd_failover_hello_check( fd_failover_hello_t const * self, @@ -27,21 +29,42 @@ fd_failover_hello_check( fd_failover_hello_t const * self, } void -fd_failover_member_cert_msg( uchar out[ 48 ], - uchar const junk_pubkey[ 32 ] ) { - fd_memcpy( out, FD_KEYGUARD_MEMBER_CERT_PREFIX, FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ ); - fd_memcpy( out+FD_KEYGUARD_MEMBER_CERT_PREFIX_SZ, junk_pubkey, 32UL ); +fd_failover_admission_hash( fd_failover_admission_start_t const * start, + uchar const * client_hello, + fd_failover_admission_challenge_t const * challenge, + uchar out[ 32 ] ) { + fd_sha256_t sha[1]; + fd_sha256_init( sha ); + fd_sha256_append( sha, start, sizeof(*start) ); + fd_sha256_append( sha, client_hello, start->client_hello_sz ); + fd_sha256_append( sha, challenge, sizeof(*challenge) ); + fd_sha256_fini( sha, out ); } -int -fd_failover_member_cert_check( fd_failover_hello_t const * hello, - fd_sha512_t * sha ) { - uchar msg[ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ]; - fd_failover_member_cert_msg( msg, hello->junk_pubkey ); - if( FD_UNLIKELY( fd_ed25519_verify( msg, sizeof(msg), hello->member_cert, hello->staked_pubkey, sha )!=FD_ED25519_SUCCESS ) ) { - return FD_FAILOVER_HELLO_ERR_CERT; - } - return FD_FAILOVER_HELLO_OK; +void +fd_failover_response_hash( uchar const transcript[ 32 ], + uchar const client_proof[ 64 ], + uchar const * server_flight, + ushort server_flight_sz, + uchar out[ 32 ] ) { + fd_sha256_t sha[1]; + fd_sha256_init( sha ); + fd_sha256_append( sha, transcript, 32UL ); + fd_sha256_append( sha, client_proof, 64UL ); + fd_sha256_append( sha, &server_flight_sz, sizeof(server_flight_sz) ); + fd_sha256_append( sha, server_flight, server_flight_sz ); + fd_sha256_fini( sha, out ); +} + +void +fd_failover_proof_msg( uchar out[ FD_KEYGUARD_FAILOVER_PROOF_SZ ], + int server, + uchar const junk_pubkey[ 32 ], + uchar const transcript[ 32 ] ) { + fd_memcpy( out, server ? FD_KEYGUARD_FAILOVER_SERVER_DOMAIN : FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN, + FD_KEYGUARD_FAILOVER_DOMAIN_SZ ); + fd_memcpy( out+FD_KEYGUARD_FAILOVER_DOMAIN_SZ, junk_pubkey, 32UL ); + fd_memcpy( out+FD_KEYGUARD_FAILOVER_DOMAIN_SZ+32UL, transcript, 32UL ); } ulong diff --git a/src/discof/failover/fd_failover_proto.h b/src/discof/failover/fd_failover_proto.h index 0fcb7dfda60..de08eb032be 100644 --- a/src/discof/failover/fd_failover_proto.h +++ b/src/discof/failover/fd_failover_proto.h @@ -5,6 +5,7 @@ #include "../../util/fd_util_base.h" #include "../../ballet/sha512/fd_sha512.h" +#include "../../disco/keyguard/fd_keyguard.h" /* Protocol version */ #define FD_FAILOVER_VERSION (1U) @@ -16,7 +17,8 @@ #define FD_FAILOVER_MSG_PROMOTE_ACK (3U) #define FD_FAILOVER_MSG_PROMOTE_REJECTED (4U) #define FD_FAILOVER_MSG_HANDOFF_RESULT (5U) -#define FD_FAILOVER_MSG_RESERVED (6U) +#define FD_FAILOVER_MSG_KEEPALIVE (6U) +#define FD_FAILOVER_MSG_RESERVED (7U) /* Sentinel for a slot field with no value */ #define FD_FAILOVER_SLOT_NULL (ULONG_MAX) @@ -61,7 +63,7 @@ #define FD_FAILOVER_HELLO_ERR_ROLE (7) #define FD_FAILOVER_HELLO_ERR_BOOT_ID (8) #define FD_FAILOVER_HELLO_ERR_MODE (9) -#define FD_FAILOVER_HELLO_ERR_CERT (10) +#define FD_FAILOVER_HELLO_ERR_PROOF (10) /* Upper bound on the consensus state payload in tower mode. A CompactTowerSync with block id and bank hash is under 512 bytes. */ @@ -77,10 +79,44 @@ struct __attribute__((packed)) fd_failover_hello { uchar mode; /* FD_FAILOVER_MODE_* consensus */ ulong boot_id; /* random nonzero value per boot to distinguish restarts */ uchar commit[ 20 ]; /* FD commit hash */ - uchar member_cert[ 64 ]; /* staked key's signature over the cert prefix and junk_pubkey */ }; typedef struct fd_failover_hello fd_failover_hello_t; -FD_STATIC_ASSERT( sizeof(fd_failover_hello_t)==192UL, wire_layout ); +FD_STATIC_ASSERT( sizeof(fd_failover_hello_t)==128UL, wire_layout ); + +/* Admission precedes TLS. START carries one complete TLS ClientHello + record after this fixed header. Both proofs cover START, that record + and CHALLENGE. The server proof additionally covers the client proof + and the exact prepared server TLS flight. + Nonces are public, fresh for each connection and consumed once. */ +#define FD_FAILOVER_ADMISSION_MAGIC (0x314d44414c494146UL) +#define FD_FAILOVER_ADMISSION_MAX (2048UL) +#define FD_FAILOVER_CLIENT_HELLO_MAX (1024UL) + +struct __attribute__((packed)) fd_failover_admission_start { + ulong magic; + fd_failover_hello_t hello; + uchar nonce[ 32 ]; + ushort client_hello_sz; +}; +typedef struct fd_failover_admission_start fd_failover_admission_start_t; +FD_STATIC_ASSERT( sizeof(fd_failover_admission_start_t)==170UL, admission_start_layout ); + +struct __attribute__((packed)) fd_failover_admission_challenge { + ulong magic; + fd_failover_hello_t hello; + uchar nonce[ 32 ]; +}; +typedef struct fd_failover_admission_challenge fd_failover_admission_challenge_t; +FD_STATIC_ASSERT( sizeof(fd_failover_admission_challenge_t)==168UL, admission_challenge_layout ); + +struct __attribute__((packed)) fd_failover_admission_response { + ushort server_flight_sz; + uchar proof[ 64 ]; +}; +typedef struct fd_failover_admission_response fd_failover_admission_response_t; +FD_STATIC_ASSERT( sizeof(fd_failover_admission_response_t)==66UL, admission_response_layout ); + +#define FD_FAILOVER_SERVER_FLIGHT_MAX (FD_FAILOVER_ADMISSION_MAX-sizeof(fd_failover_admission_response_t)) /* Sent by the demoter once it runs the junk key, the final tower follows it. */ @@ -145,19 +181,27 @@ int fd_failover_hello_check( fd_failover_hello_t const * self, fd_failover_hello_t const * peer ); -/* fd_failover_member_cert_msg writes the 48 byte message a member - certificate signs, the keyguard's member cert prefix then - junk_pubkey. The sign tile signs exactly this with the staked key. */ +/* Admission hash covers exact packed wire bytes and the prepared TLS + record. response_hash binds the listener proof to the client proof + and the server's prepared TLS flight, including its ephemeral key. */ void -fd_failover_member_cert_msg( uchar out[ 48 ], - uchar const junk_pubkey[ 32 ] ); +fd_failover_admission_hash( fd_failover_admission_start_t const * start, + uchar const * client_hello, + fd_failover_admission_challenge_t const * challenge, + uchar out[ 32 ] ); -/* fd_failover_member_cert_check returns FD_FAILOVER_HELLO_OK if the - member_cert in hello is the staked pubkey's signature over the junk - pubkey in hello, else FD_FAILOVER_HELLO_ERR_CERT. */ -int -fd_failover_member_cert_check( fd_failover_hello_t const * hello, - fd_sha512_t * sha ); +void +fd_failover_response_hash( uchar const transcript[ 32 ], + uchar const client_proof[ 64 ], + uchar const * server_flight, + ushort server_flight_sz, + uchar out[ 32 ] ); + +void +fd_failover_proof_msg( uchar out[ FD_KEYGUARD_FAILOVER_PROOF_SZ ], + int server, + uchar const junk_pubkey[ 32 ], + uchar const transcript[ 32 ] ); /* fd_failover_session_init returns the resting state of an endpoint, LISTENING for the listener and BACKOFF for the dialer. */ diff --git a/src/discof/failover/fd_failover_tile.c b/src/discof/failover/fd_failover_tile.c index c05a7ea3c5b..4b904bca9ff 100644 --- a/src/discof/failover/fd_failover_tile.c +++ b/src/discof/failover/fd_failover_tile.c @@ -14,6 +14,7 @@ #include "fd_failover_bus.h" #include "fd_failover_channel.h" +#include "../../disco/metrics/fd_metrics.h" #include #include @@ -22,7 +23,7 @@ /* The failov tile owns the sockets to the other failover machine, keeps host networking for them and holds no private key. The sign tile - signs our member certificate with the failover identity and our TLS + signs fresh admission proofs with the failover identity and our TLS handshakes with the junk key, and the admin tile does the identity switch and stays off the network. */ @@ -195,9 +196,9 @@ struct fd_failover_id_switch { typedef struct fd_failover_id_switch fd_failover_id_switch_t; struct fd_failover_tile_ctx { - fd_keyguard_client_t keyguard_client[ 1 ]; /* signs the member certificate */ - int member_cert_set; /* hello carries a signed member certificate */ + fd_keyguard_client_t keyguard_client[ 1 ]; /* signs admission proofs and TLS */ int client_only; /* no listener and no incoming handoffs */ + int input_lost; /* input overrun disables failover until restart */ int target_pending; /* waiting for Admin to prepare the client's target */ fd_failover_hello_t hello; /* the HELLO we send */ @@ -216,7 +217,7 @@ struct fd_failover_tile_ctx { ulong peer_floor; /* highest last vote the peer reported, kept across sessions */ ulong own_floor; /* highest vote this machine signed since it started */ - /* gossip_ciseen, reliable contact info updates */ + /* gossip_ciseen, contact info updates */ ulong gossip_in_idx; /* gossip_ciseen in link */ fd_wksp_t * gossip_in_mem; /* its workspace */ ulong gossip_in_chunk0; /* its first chunk */ @@ -324,6 +325,14 @@ sign_ed25519( void * signer_ctx, fd_keyguard_client_sign( ctx->keyguard_client, sig, msg, FD_TLS_CV_SIGN_SZ, FD_KEYGUARD_SIGN_TYPE_ED25519 ); } +static void +sign_proof( void * signer_ctx, + uchar sig[ 64 ], + uchar const msg[ FD_KEYGUARD_FAILOVER_PROOF_SZ ] ) { + fd_failover_tile_ctx_t * ctx = signer_ctx; + fd_keyguard_client_sign( ctx->keyguard_client, sig, msg, FD_KEYGUARD_FAILOVER_PROOF_SZ, FD_KEYGUARD_SIGN_TYPE_ED25519 ); +} + static void privileged_init( fd_topo_t const * topo, fd_topo_tile_t const * tile ) { @@ -396,7 +405,7 @@ privileged_init( fd_topo_t const * topo, ctx->channel = fd_failover_channel_join( fd_failover_channel_new( channel_mem ) ); FD_TEST( ctx->channel ); fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 }; - if( FD_UNLIKELY( !ctx->target_pending && fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, &ctx->hello ) ) ) { + if( FD_UNLIKELY( !ctx->target_pending && fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, (fd_failover_proof_signer_t){ .ctx=ctx, .sign_fn=sign_proof }, &ctx->hello ) ) ) { FD_LOG_ERR(( "failover TLS setup failed for the junk identity `%s`", tile->failov.identity_key_path )); } @@ -502,30 +511,6 @@ unprivileged_init( fd_topo_t const * topo, } } -/* Installs the member certificate the sign tile signed, unless it was - signed with a key other than the failover identity. */ -static void -member_cert_signed( fd_failover_tile_ctx_t * ctx, - uchar const cert[ 64 ] ) { - ctx->member_cert_set = 1; - if( FD_UNLIKELY( fd_failover_channel_set_member_cert( ctx->channel, cert ) ) ) { - FD_LOG_WARNING(( "the sign tile signed our member certificate with another key, set-identity changed the failover identity, we ask again once failover restarts for it" )); - return; - } - FD_LOG_NOTICE(( "the sign tile signed our member certificate with the failover identity, this machine can pair with the other failover machine now" )); -} - -/* Asks the sign tile to sign our member certificate with the failover - identity, and installs it. */ -static void -request_member_cert( fd_failover_tile_ctx_t * ctx ) { - uchar msg [ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ]; - uchar cert[ 64 ]; - fd_failover_member_cert_msg( msg, ctx->hello.junk_pubkey ); - fd_keyguard_client_sign( ctx->keyguard_client, cert, msg, sizeof(msg), FD_KEYGUARD_SIGN_TYPE_ED25519 ); - member_cert_signed( ctx, cert ); -} - /* Updates our session state when the channel's session changes. */ static void sync_session( fd_failover_tile_ctx_t * ctx ) { @@ -976,10 +961,9 @@ operator_switched( fd_failover_tile_ctx_t * ctx, ctx->role = staked ? FD_FAILOVER_ROLE_ACTIVE : FD_FAILOVER_ROLE_STANDBY; ctx->hello.role = (uchar)ctx->role; fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 }; - if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, &ctx->hello ) ) ) { + if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, (fd_failover_proof_signer_t){ .ctx=ctx, .sign_fn=sign_proof }, &ctx->hello ) ) ) { FD_LOG_ERR(( "failover TLS setup failed after set-identity" )); } - ctx->member_cert_set = 0; ctx->target_pending = 0; sync_session( ctx ); @@ -1127,6 +1111,9 @@ static char const * control_refusal( ulong result, char const ** hint ) { switch( result ) { + case FD_FAILOVER_CONTROL_RESULT_INPUT_LOST: + *hint = "failover lost an input update and is disabled until restart, validator processing continues"; + return "INPUT_LOST"; case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: *hint = "`failover promote` runs on a standby, see `failover status`"; return "BAD_ROLE"; @@ -1567,6 +1554,13 @@ step_promote_switch( fd_failover_tile_ctx_t * ctx ) { reject_promotion( ctx, FD_FAILOVER_REJECT_SWITCH_FAILED, 0 ); return; } + /* A local recovery changes the role outside a handoff. Reauthenticate + so a background session cannot retain our old standby HELLO. */ + if( !ctx->promotion.from_peer ) { + fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); + fd_failover_channel_hangup( ctx->channel, fd_failover_clock() ); + sync_session( ctx ); + } /* Admin confirmed the staked identity is installed, become active. */ set_role( ctx, FD_FAILOVER_ROLE_ACTIVE ); /* A new tenure starts, the peer tower we kept is older than its votes. */ @@ -1799,9 +1793,9 @@ handle_control( fd_failover_tile_ctx_t * ctx, /* A bound final result ends the request without changing our installed identity. */ request_end( ctx, now, failed ? FD_FAILOVER_HANDOFF_DECLINED : FD_FAILOVER_HANDOFF_TAKEN ); if( !failed ) - FD_LOG_NOTICE(( "handoff %lu complete: the old active confirmed our ACK, we hold the staked identity, on-demand connection closed", result.handoff_id )); + FD_LOG_NOTICE(( "handoff %lu complete: the old active confirmed our ACK, we hold the staked identity, connection closed", result.handoff_id )); else - FD_LOG_NOTICE(( "handoff %lu: request ended, %s, on-demand connection closed", result.handoff_id, + FD_LOG_NOTICE(( "handoff %lu: request ended, %s, connection closed", result.handoff_id, ctx->role==FD_FAILOVER_ROLE_ACTIVE ? "we still hold the staked identity" : "we remain standby" )); return; } @@ -1922,11 +1916,33 @@ request_end( fd_failover_tile_ctx_t * ctx, sync_session( ctx ); } -/* The standby dials only while its operator's handoff is open. */ +/* An idle standby authenticates ahead of an operator command. Merely + pairing never creates a request or changes the installed identity. */ +static void +preconnect( fd_failover_tile_ctx_t * ctx, + long now ) { + if( ctx->request.id || ctx->stuck || ctx->tx.valid || ctx->session.close_after_send || + ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) return; + /* A stale gossip address may name a validator that now boots passive. + Do not keep that standby session open. Retain the dial backoff. */ + if( ctx->role==FD_FAILOVER_ROLE_STANDBY && + fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED && + fd_failover_channel_peer_hello( ctx->channel )->role!=FD_FAILOVER_ROLE_ACTIVE ) + fd_failover_channel_hangup( ctx->channel, now ); + int ready = !ctx->target_pending && ctx->role==FD_FAILOVER_ROLE_STANDBY && + ctx->staked_addr && ctx->staked_seen_at && now>=ctx->staked_seen_at && + now-ctx->staked_seen_atchannel, 0U, 0 ); + else if( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ) + fd_failover_channel_init_dialer( ctx->channel, ctx->staked_addr, ctx->port ); +} + +/* An operator command is required before sending a handoff request. */ static void peer_poll( fd_failover_tile_ctx_t * ctx, long now, int * charge_busy ) { + preconnect( ctx, now ); sync_session( ctx ); if( FD_UNLIKELY( ctx->request.id && ctx->request.until && now>=ctx->request.until && ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || @@ -1941,7 +1957,7 @@ peer_poll( fd_failover_tile_ctx_t * ctx, /* The result drained, close the session without changing role or action. */ ctx->session.close_after_send = 0; fd_failover_channel_hangup( ctx->channel, now ); - FD_LOG_NOTICE(( "handoff %lu: confirmation drained, on-demand connection closed, listener ready for the next request", ctx->session.close_handoff_id )); + FD_LOG_NOTICE(( "handoff %lu: confirmation drained, connection closed, listener ready for the next request", ctx->session.close_handoff_id )); sync_session( ctx ); return; } @@ -2015,6 +2031,7 @@ static ulong promote_guard( fd_failover_tile_ctx_t const * ctx, long now, int force ) { + if( FD_UNLIKELY( ctx->input_lost ) ) return FD_FAILOVER_CONTROL_RESULT_INPUT_LOST; int paired = fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED; if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_STANDBY ) ) return FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; @@ -2061,6 +2078,7 @@ static ulong apply_control( fd_failover_tile_ctx_t * ctx, fd_adminctl_failover_req_t const * req, long now ) { + if( FD_UNLIKELY( ctx->input_lost ) ) return FD_FAILOVER_CONTROL_RESULT_INPUT_LOST; if( FD_UNLIKELY( ctx->target_pending && req->cmd!=FD_ADMINCTL_FAILOVER_CMD_STATUS ) ) return FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED; switch( req->cmd ) { @@ -2073,15 +2091,15 @@ apply_control( fd_failover_tile_ctx_t * ctx, paused = 0; } if( FD_UNLIKELY( paused ) ) { - /* Resume the paused request, a new --address or --port replaces - where we dial. */ + /* Refresh the destination while retaining the authenticated member + and boot. An explicit address takes precedence over gossip. */ ctx->request.until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); ctx->request.sent = 0; ctx->stuck = 0; if( req->addr ) ctx->cmd_addr = req->addr; if( req->port ) ctx->cmd_port = req->port; - if( ctx->cmd_addr ) ctx->request.addr = ctx->cmd_addr; - else if( !ctx->request.peer.boot_id && ctx->staked_addr ) ctx->request.addr = ctx->staked_addr; + if( ctx->cmd_addr ) ctx->request.addr = ctx->cmd_addr; + else if( ctx->staked_addr ) ctx->request.addr = ctx->staked_addr; fd_failover_channel_init_dialer( ctx->channel, ctx->request.addr, dial_port( ctx ) ); FD_LOG_NOTICE(( "resuming handoff %lu at `" FD_IP4_ADDR_FMT ":%hu` from %s, %s, %s", ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), dial_port( ctx ), @@ -2185,7 +2203,8 @@ status_snapshot( fd_failover_tile_ctx_t const * ctx, resp->handoff_result = (uchar)( ctx->last_requested ? ctx->request.result : ctx->handoff.result ); resp->promote_source = (uchar)promote_source( ctx ); resp->promote_floor = coverage_floor( ctx ); - resp->promote_result = ctx->target_pending ? FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED : promote_guard( ctx, now, 0 ); + resp->promote_result = ctx->input_lost ? FD_FAILOVER_CONTROL_RESULT_INPUT_LOST : + ctx->target_pending ? FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED : promote_guard( ctx, now, 0 ); resp->mode = ctx->hello.mode; resp->request_paused = (uchar)request_paused( ctx ); } @@ -2264,11 +2283,26 @@ gossip_commit( fd_failover_tile_ctx_t * ctx, } } +/* Failover must not backpressure validator work. Losing an input instead + disables transitions because cached history or switch results may be stale. */ +static void +input_overrun( fd_failover_tile_ctx_t * ctx ) { + if( FD_UNLIKELY( ctx->input_lost ) ) return; + ctx->input_lost = 1; + ctx->stuck = 1; + ctx->slot_done_fresh = 0; + ctx->bus_req_fresh = 0; + fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); + fd_failover_channel_hangup( ctx->channel, fd_failover_clock() ); + FD_LOG_WARNING(( "failover input overrun, failover is disabled until restart, validator processing continues" )); +} + static inline int before_frag( fd_failover_tile_ctx_t * ctx, ulong in_idx, ulong seq, ulong sig ) { + if( FD_UNLIKELY( ctx->input_lost ) ) return in_idx!=ctx->admin_in_idx || sig!=FD_FAILOVER_BUS_REQUEST; if( FD_LIKELY( in_idx==ctx->gossip_in_idx ) ) { return sig!=FD_GOSSIP_UPDATE_TAG_CONTACT_INFO && sig!=FD_GOSSIP_UPDATE_TAG_CONTACT_INFO_REMOVE; } @@ -2391,7 +2425,14 @@ after_credit( fd_failover_tile_ctx_t * ctx, fd_stem_context_t * stem, int * opt_poll_in FD_PARAM_UNUSED, int * charge_busy ) { - if( FD_UNLIKELY( !ctx->target_pending && !ctx->member_cert_set ) ) request_member_cert( ctx ); + if( FD_UNLIKELY( ctx->input_lost ) ) { + if( FD_UNLIKELY( ctx->bus_req_fresh ) ) { + ctx->bus_req_fresh = 0; + serve_bus_request( ctx, stem, fd_failover_clock() ); + *charge_busy = 1; + } + return; + } if( FD_UNLIKELY( ctx->slot_done_fresh ) ) { ctx->slot_done_fresh = 0; consume_slot_done( ctx, &ctx->slot_done ); @@ -2453,6 +2494,26 @@ populate_allowed_fds( fd_topo_t const * topo, return out_cnt; } +static void +metrics_write( fd_failover_tile_ctx_t * ctx ) { + fd_failover_channel_metrics_t const * m = fd_failover_channel_metrics( ctx->channel ); + FD_MCNT_SET( FAILOV, CONN_ACCEPTED, m->accept_cnt ); + FD_MCNT_SET( FAILOV, ACCEPT_BUDGET_EXHAUSTED, m->accept_limited_cnt ); + FD_MCNT_SET( FAILOV, CONN_REJECTED_SOURCE, m->source_reject_cnt ); + FD_MCNT_SET( FAILOV, CONN_REJECTED_BUDGET, m->budget_reject_cnt ); + FD_MCNT_SET( FAILOV, CONN_REJECTED_CAPACITY, m->capacity_reject_cnt ); + FD_MCNT_SET( FAILOV, CONN_RECONNECT_ADMITTED, m->reconnect_cnt ); + FD_MCNT_SET( FAILOV, HANDSHAKE_TIMED_OUT, m->handshake_timeout_cnt ); + FD_MCNT_SET( FAILOV, PROOF_REJECTED, m->proof_reject_cnt ); + FD_MCNT_SET( FAILOV, HELLO_REJECTED, m->hello_reject_cnt ); + FD_MCNT_SET( FAILOV, TLS_HANDSHAKE_FAILED, m->tls_fail_cnt ); + FD_MCNT_SET( FAILOV, FRAME_PROTOCOL_REJECTED, m->wire_fatal_cnt ); + FD_MCNT_SET( FAILOV, CONN_AUTHENTICATED, m->paired_cnt ); + FD_MCNT_SET( FAILOV, HANDSHAKE_DURATION_NANOS, m->handshake_duration_nanos ); + FD_MGAUGE_SET( FAILOV, HANDSHAKE_LATEST_DURATION_NANOS, m->handshake_latest_nanos ); + FD_MGAUGE_SET( FAILOV, CONN_PENDING, fd_failover_channel_pending( ctx->channel ) ); +} + /* An iteration publishes at most an adopt or switch request and a bus response, within the burst of 3. */ #define STEM_BURST (3UL) @@ -2461,13 +2522,16 @@ populate_allowed_fds( fd_topo_t const * topo, /* A frame on our sockets does not wake a parked tile, so we do not park. */ #define STEM_NEVER_PARK 1 -#define STEM_CALLBACK_CONTEXT_TYPE fd_failover_tile_ctx_t -#define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_failover_tile_ctx_t) +#define STEM_CALLBACK_CONTEXT_TYPE fd_failover_tile_ctx_t +#define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_failover_tile_ctx_t) -#define STEM_CALLBACK_AFTER_CREDIT after_credit -#define STEM_CALLBACK_BEFORE_FRAG before_frag -#define STEM_CALLBACK_DURING_FRAG during_frag -#define STEM_CALLBACK_AFTER_FRAG after_frag +#define STEM_CALLBACK_METRICS_WRITE metrics_write +#define STEM_CALLBACK_AFTER_POLL_OVERRUN input_overrun +#define STEM_CALLBACK_AFTER_READ_OVERRUN input_overrun +#define STEM_CALLBACK_AFTER_CREDIT after_credit +#define STEM_CALLBACK_BEFORE_FRAG before_frag +#define STEM_CALLBACK_DURING_FRAG during_frag +#define STEM_CALLBACK_AFTER_FRAG after_frag #include "../../disco/stem/fd_stem.c" diff --git a/src/discof/failover/fd_failover_tls.c b/src/discof/failover/fd_failover_tls.c index ecd3d3b9be9..a131fcd1ae3 100644 --- a/src/discof/failover/fd_failover_tls.c +++ b/src/discof/failover/fd_failover_tls.c @@ -4,6 +4,8 @@ #include "../../ballet/x509/fd_x509_mock.h" #include +#include +#include /* ALPN id, TLS length-prefixed */ static uchar const ALPN[] = "\x17" "fd-failover/on-demand/1"; @@ -69,14 +71,52 @@ fd_failover_tls_new( fd_failover_tls_t * tls, return 0; } +int +fd_failover_tls_client_hello( fd_failover_tls_t * tls, uchar * out, ulong * out_sz ) { + ulong app_sz = 0UL; + return fd_tlsrec_conn_rx( &tls->conn, NULL, out, out_sz, NULL, &app_sz ); +} + +int +fd_failover_tls_accept_flight( fd_failover_tls_t * tls, uchar const * data, ulong sz ) { + if( fd_tlsrec_sock_tx_pending( &tls->sock ) || tls->received>FD_FAILOVER_TLS_PREPAIR_MAX || + sz>FD_FAILOVER_TLS_PREPAIR_MAX-tls->received ) return -1; + fd_tlsrec_slice_t rx[1]; + fd_tlsrec_slice_init( rx, (uchar *)data, sz ); + ulong tx_sz = sizeof(tls->sock.tx_buf); + ulong app_sz = 0UL; + int rc = fd_tlsrec_conn_rx( &tls->conn, rx, tls->sock.tx_buf, &tx_sz, NULL, &app_sz ); + tls->sock.tx_sz = tx_sz; + tls->received += sz; + return rc || !fd_tlsrec_slice_is_empty( rx ) || app_sz ? -1 : 0; +} + +int +fd_failover_tls_flush( fd_failover_tls_t * tls ) { + fd_tlsrec_sock_t * sock = &tls->sock; + if( !fd_tlsrec_sock_tx_pending( sock ) ) return 0; + if( !tls->send_budget ) return 1; + tls->send_budget--; + long sent = send( tls->fd, sock->tx_buf+sock->tx_off, sock->tx_sz-sock->tx_off, MSG_NOSIGNAL|MSG_DONTWAIT ); + if( sent<0L ) return errno==EINTR || errno==EAGAIN || errno==EWOULDBLOCK ? 1 : -1; + if( FD_UNLIKELY( !sent ) ) return -1; + sock->tx_off += (ulong)sent; + if( sock->tx_offtx_sz ) return 1; + sock->tx_off = sock->tx_sz = 0UL; + return 0; +} + void fd_failover_tls_fini( fd_failover_tls_t * tls ) { if( FD_LIKELY( tls->fd!=-1 && fd_tlsrec_conn_is_ready( &tls->conn ) && !tls->conn.tx_closed ) ) { - (void)fd_tlsrec_sock_close( &tls->sock, &tls->conn, tls->fd ); + ulong sz = sizeof(tls->sock.tx_buf)-tls->sock.tx_sz; + if( !fd_tlsrec_conn_close( &tls->conn, tls->sock.tx_buf+tls->sock.tx_sz, &sz ) ) { + tls->sock.tx_sz += sz; + (void)fd_failover_tls_flush( tls ); + } } fd_memzero_explicit( &tls->conn, sizeof(tls->conn) ); fd_memzero_explicit( &tls->sock, sizeof(tls->sock) ); - /* Closing clears both TLS verification and HELLO pairing. */ tls->fd = -1; tls->verified = 0; tls->paired = 0; @@ -87,25 +127,38 @@ void fd_failover_tls_budget( fd_failover_tls_t * tls ) { tls->read_budget = 1; tls->write_budget = 1; + tls->send_budget = 1; } -/* pull reads one batch of ciphertext into the record layer. Returns 0 - (maybe nothing read) or -1 on error or peer close. */ - +/* Limit ciphertext before processing it. EINTR yields to the next + service turn, so neither reads nor writes retry in an unbounded loop. */ static int pull( fd_failover_tls_t * tls ) { - if( FD_UNLIKELY( !tls->read_budget ) ) return 0; - tls->read_budget--; - ulong rx_sz = 0UL; - int rc = fd_tlsrec_sock_rx( &tls->sock, &tls->conn, tls->fd, &rx_sz ); - if( FD_UNLIKELY( rc ) ) { - tls->peer_closed = rc==FD_TLSREC_SOCK_ERR_EOF; - return -1; - } - if( FD_LIKELY( !tls->paired ) ) { - tls->received += rx_sz; - if( FD_UNLIKELY( tls->received>FD_FAILOVER_TLS_PREPAIR_MAX ) ) return -1; + fd_tlsrec_sock_t * sock = &tls->sock; + if( !tls->read_budget || fd_tlsrec_sock_rx_avail( sock ) || + sizeof(sock->tx_buf)-sock->tx_szconn.rx_closed ) { tls->peer_closed = 1; return -1; } + ulong max = FD_TLSREC_SOCK_RX_MTU; + if( !tls->paired ) { + if( tls->received>=FD_FAILOVER_TLS_PREPAIR_MAX ) return -1; + max = fd_ulong_min( max, FD_FAILOVER_TLS_PREPAIR_MAX-tls->received ); } + tls->read_budget--; + uchar buf[ FD_TLSREC_SOCK_RX_MTU ]; + long n = recv( tls->fd, buf, max, MSG_NOSIGNAL|MSG_DONTWAIT ); + if( n<0L ) return errno==EINTR || errno==EAGAIN || errno==EWOULDBLOCK ? 0 : -1; + if( !n ) { tls->peer_closed = 1; return -1; } + if( !tls->paired ) tls->received += (ulong)n; + fd_tlsrec_slice_t rx[1]; + fd_tlsrec_slice_init( rx, buf, (ulong)n ); + ulong tx_sz = sizeof(sock->tx_buf)-sock->tx_sz; + ulong app_sz = sizeof(sock->rx_buf); + int rc = fd_tlsrec_conn_rx( &tls->conn, rx, sock->tx_buf+sock->tx_sz, &tx_sz, sock->rx_buf, &app_sz ); + sock->tx_sz += tx_sz; + sock->rx_off = 0UL; + sock->rx_sz = app_sz; + if( rc || fd_failover_tls_flush( tls )<0 ) return -1; + if( tls->conn.rx_closed && !app_sz ) { tls->peer_closed = 1; return -1; } return 0; } @@ -117,7 +170,7 @@ fd_failover_tls_handshake( fd_failover_tls_t * tls ) { FD_LOG_DEBUG(( "failover TLS handshake failed: %s", fd_tls_reason_cstr( tls->conn.hs.base.reason ) )); return -1; } - if( FD_UNLIKELY( fd_tlsrec_sock_flush( &tls->sock, tls->fd )<0 ) ) return -1; + if( FD_UNLIKELY( fd_failover_tls_flush( tls )<0 ) ) return -1; if( FD_LIKELY( !fd_tlsrec_conn_is_ready( &tls->conn ) ) ) { if( FD_UNLIKELY( pull( tls ) ) ) return -1; if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) ) ) { @@ -156,7 +209,7 @@ fd_failover_tls_read( fd_failover_tls_t * tls, tls->peer_closed = 1; return -1L; } - if( FD_UNLIKELY( fd_tlsrec_sock_flush( &tls->sock, tls->fd )<0 ) ) return -1L; + if( FD_UNLIKELY( fd_failover_tls_flush( tls )<0 ) ) return -1L; if( FD_UNLIKELY( pull( tls ) ) ) return -1L; } return (long)fd_tlsrec_sock_rx_pop( &tls->sock, buf, sz ); @@ -168,12 +221,17 @@ fd_failover_tls_write( fd_failover_tls_t * tls, ulong sz ) { if( FD_UNLIKELY( fd_tlsrec_conn_is_failed( &tls->conn ) || tls->conn.tx_closed ) ) return -1L; if( FD_UNLIKELY( fd_tlsrec_sock_tx_pending( &tls->sock ) ) ) { - int rc = fd_tlsrec_sock_flush( &tls->sock, tls->fd ); + int rc = fd_failover_tls_flush( tls ); if( rc ) return rc<0 ? -1L : 0L; } if( FD_UNLIKELY( !tls->write_budget ) ) return 0L; tls->write_budget--; - ulong consumed = 0UL; - if( FD_UNLIKELY( fd_tlsrec_sock_tx( &tls->sock, &tls->conn, tls->fd, buf, sz, &consumed ) ) ) return -1L; - return (long)consumed; + fd_tlsrec_slice_t app[1]; + fd_tlsrec_slice_init( app, (uchar *)buf, sz ); + ulong tx_sz = sizeof(tls->sock.tx_buf); + if( FD_UNLIKELY( fd_tlsrec_conn_tx( &tls->conn, tls->sock.tx_buf, &tx_sz, app ) ) ) return -1L; + tls->sock.tx_off = 0UL; + tls->sock.tx_sz = tx_sz; + if( FD_UNLIKELY( fd_failover_tls_flush( tls )<0 ) ) return -1L; + return (long)(sz-fd_tlsrec_slice_sz( app )); } diff --git a/src/discof/failover/fd_failover_tls.h b/src/discof/failover/fd_failover_tls.h index cca4bef6122..b963b5b8f74 100644 --- a/src/discof/failover/fd_failover_tls.h +++ b/src/discof/failover/fd_failover_tls.h @@ -3,8 +3,8 @@ /* TLS 1.3 over TCP for one failover candidate, built on fd_tls and fd_tlsrec. Both ends present an Ed25519 cert for their junk identity. - TLS does not check the peer key against an expected key, the channel - checks it against HELLO and its member certificate. The caller owns + The channel admits a peer before TLS and checks its TLS key + against the admitted HELLO. The caller owns the socket. */ #include "../../waltz/tlsrec/fd_tlsrec_sock.h" @@ -35,6 +35,7 @@ struct fd_failover_tls { uchar peer_pubkey[ 32 ]; /* junk key the peer signed the handshake with */ int read_budget; /* socket reads left this turn */ int write_budget; /* record writes left this turn */ + int send_budget; /* socket sends left this turn */ ulong received; /* ciphertext bytes read before pairing */ }; typedef struct fd_failover_tls fd_failover_tls_t; @@ -66,6 +67,19 @@ fd_failover_tls_new( fd_failover_tls_t * tls, int fd, int dial_peer ); +/* Prepare the outbound ClientHello without sending it, or feed an + admitted peer flight to the connection without socket I/O. The + generated reply is left in sock.tx_buf for the caller to capture or + flush. No earlier output may be pending. */ +int +fd_failover_tls_client_hello( fd_failover_tls_t * tls, uchar * out, ulong * out_sz ); + +int +fd_failover_tls_accept_flight( fd_failover_tls_t * tls, uchar const * data, ulong sz ); + +int +fd_failover_tls_flush( fd_failover_tls_t * tls ); + /* fd_failover_tls_fini sends close_notify if the connection is up and wipes the connection state. The caller closes fd. */ diff --git a/src/discof/failover/fd_failover_wire.c b/src/discof/failover/fd_failover_wire.c index 08b687f859f..bc6d2d8d9a9 100644 --- a/src/discof/failover/fd_failover_wire.c +++ b/src/discof/failover/fd_failover_wire.c @@ -2,6 +2,16 @@ #include +static ulong const payload_max[ FD_FAILOVER_MSG_RESERVED ] = { + sizeof(fd_failover_hello_t), + sizeof(fd_failover_handoff_request_t), + FD_FAILOVER_DEMOTED_PAYLOAD_MAX, + sizeof(fd_failover_promote_ack_t), + sizeof(fd_failover_promote_rejected_t), + sizeof(fd_failover_handoff_result_t), + 0UL /* KEEPALIVE */ +}; + void fd_failover_wire_session_init( fd_failover_wire_session_t * session ) { session->tx_seq = 0UL; @@ -15,7 +25,7 @@ fd_failover_wire_encode( fd_failover_wire_session_t * session, uchar const * payload, ulong payload_sz ) { if( FD_UNLIKELY( type>=FD_FAILOVER_MSG_RESERVED || - payload_sz>FD_FAILOVER_PAYLOAD_MAX || + payload_sz>payload_max[type] || session->tx_seq==ULONG_MAX ) ) return 0UL; ulong frame_sz = FD_FAILOVER_FRAME_HDR_SZ+payload_sz; @@ -49,7 +59,6 @@ fd_failover_wire_decode( fd_failover_wire_session_t * session, ulong frame_sz = (ulong)hdr.len; if( FD_UNLIKELY( frame_szFD_FAILOVER_FRAME_MAX ) ) return FD_FAILOVER_WIRE_ERR_SZ; - if( FD_UNLIKELY( buf_szrx_seq==ULONG_MAX || hdr.seq!=session->rx_seq ) ) return FD_FAILOVER_WIRE_ERR_SEQ; if( FD_UNLIKELY( hdr.type>=FD_FAILOVER_MSG_RESERVED ) ) return FD_FAILOVER_WIRE_ERR_TYPE; + if( FD_UNLIKELY( payload_sz>payload_max[hdr.type] ) ) return FD_FAILOVER_WIRE_ERR_SZ; + if( FD_UNLIKELY( buf_szrx_seq++; *out_type = hdr.type; *out_payload = buf+FD_FAILOVER_FRAME_HDR_SZ; diff --git a/src/discof/failover/test_failover_channel.c b/src/discof/failover/test_failover_channel.c index 97580598401..38a73ffcc47 100644 --- a/src/discof/failover/test_failover_channel.c +++ b/src/discof/failover/test_failover_channel.c @@ -3,7 +3,7 @@ #include "../../ballet/ed25519/fd_ed25519.h" #include "../../util/net/fd_ip4.h" -static uchar scratch[ 2 ][ 8UL<<20 ] __attribute__((aligned(128))); +static uchar scratch[ 2 ][ 32UL<<20 ] __attribute__((aligned(128))); static uchar keys[ 3 ][ 64 ] = { { 1 }, { 2 }, { 9 } }; static fd_sha512_t sha[ 1 ]; @@ -16,6 +16,12 @@ sign_fn( void * ctx, fd_ed25519_sign( sig, payload, FD_TLS_CV_SIGN_SZ, key+32UL, key, sha ); } +static void +proof_fn( void * ctx, uchar sig[ 64 ], uchar const msg[ FD_KEYGUARD_FAILOVER_PROOF_SZ ] ) { + uchar const * key = ctx; + fd_ed25519_sign( sig, msg, FD_KEYGUARD_FAILOVER_PROOF_SZ, key+32UL, key, sha ); +} + static fd_failover_channel_t * member( ulong idx, ulong staked_idx ) { fd_failover_channel_t * ch = fd_failover_channel_join( fd_failover_channel_new( scratch[idx] ) ); @@ -26,11 +32,8 @@ member( ulong idx, ulong staked_idx ) { fd_memcpy( hello.junk_pubkey, keys[idx]+32UL, 32UL ); fd_memcpy( hello.staked_pubkey, keys[staked_idx]+32UL, 32UL ); fd_memset( hello.vote_account, 0xBB, 32UL ); - FD_TEST( !fd_failover_channel_set_identity( ch, keys[idx]+32UL, (fd_tls_sign_t){ .ctx=keys[idx], .sign_fn=sign_fn }, &hello ) ); - uchar msg[ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ], cert[ 64 ]; - fd_failover_member_cert_msg( msg, keys[idx]+32UL ); - fd_ed25519_sign( cert, msg, sizeof(msg), keys[staked_idx]+32UL, keys[staked_idx], sha ); - FD_TEST( !fd_failover_channel_set_member_cert( ch, cert ) ); + FD_TEST( !fd_failover_channel_set_identity( ch, keys[idx]+32UL, (fd_tls_sign_t){ .ctx=keys[idx], .sign_fn=sign_fn }, + (fd_failover_proof_signer_t){ .ctx=keys[staked_idx], .sign_fn=proof_fn }, &hello ) ); fd_failover_channel_init_listener( ch, FD_IP4_ADDR(127,0,0,1), 0 ); return ch; } diff --git a/src/discof/failover/test_failover_tile.c b/src/discof/failover/test_failover_tile.c index 8bd61a9d482..0367f2c431c 100644 --- a/src/discof/failover/test_failover_tile.c +++ b/src/discof/failover/test_failover_tile.c @@ -32,7 +32,7 @@ stem_init( void ) { }; } -static uchar ch_mem[ 8UL<<20 ] __attribute__((aligned(128))); +static uchar ch_mem[ 32UL<<20 ] __attribute__((aligned(128))); static fd_failover_tile_ctx_t ctl[ 1 ]; #define OUR_BOOT_ID (1000UL) @@ -272,7 +272,7 @@ test_operator_switch( void ) { FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && ctx->hello.role==FD_FAILOVER_ROLE_STANDBY ); FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->stuck && !ctx->tx.valid && !ctx->handoff.id && !ctx->request.id ); FD_TEST( ctx->hello.boot_id!=OUR_BOOT_ID && ctx->channel->self_hello.boot_id==ctx->hello.boot_id ); - FD_TEST( !ctx->member_cert_set && !ctx->current_tower.valid && ctx->own_floor==99UL ); + FD_TEST( !ctx->current_tower.valid && ctx->own_floor==99UL ); FD_TEST( ctx->id_switch.epoch==1UL && ctx->id_switch.discard && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK ); FD_TEST( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ); @@ -320,8 +320,7 @@ test_operator_switch( void ) { } /* test_operator_failover_identity: HELLO has the failover identity - OPERATOR gives, the role follows the installed key, and a member cert - signed with another key waits for the next OPERATOR. */ + OPERATOR gives, and the role follows the installed key. */ static void test_operator_failover_identity( void ) { fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); @@ -331,7 +330,7 @@ test_operator_failover_identity( void ) { uchar third[ 32 ]; fd_memset( third, 0x77, 32UL ); FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->hello.role==FD_FAILOVER_ROLE_ACTIVE ); FD_TEST( fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) && fd_memeq( ctx->channel->self_hello.staked_pubkey, third, 32UL ) ); - FD_TEST( !ctx->staked_addr && !ctx->staked_seen_at && !ctx->member_cert_set ); + FD_TEST( !ctx->staked_addr && !ctx->staked_seen_at ); /* A junk set-identity keeps the failover identity, the configured key does not come back. */ operator_frame( ctx, 2UL, 0x11, 0x77 ); @@ -342,32 +341,9 @@ test_operator_failover_identity( void ) { operator_frame( ctx, 3UL, 0x5A, 0x77 ); FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) ); - /* A cert over our junk key signed with the configured key, as the sign - tile would sign it before this OPERATOR, is refused by the channel and - we wait for the next OPERATOR. */ - uchar secret[ 32 ]; fd_memset( secret, 0x5A, 32UL ); - uchar public[ 32 ]; - fd_sha512_t sha[ 1 ]; - FD_TEST( fd_sha512_join( fd_sha512_new( sha ) ) ); - fd_ed25519_public_from_private( public, secret, sha ); - uchar msg[ FD_KEYGUARD_MEMBER_CERT_MSG_SZ ], cert[ 64 ]; - fd_failover_member_cert_msg( msg, ctx->hello.junk_pubkey ); - fd_ed25519_sign( cert, msg, sizeof(msg), public, secret, sha ); - member_cert_signed( ctx, cert ); - FD_TEST( ctx->member_cert_set && !ctx->channel->member_cert_set ); - - /* With a real key other than junk and staked in the next OPERATOR, a - cert it signed is ours, and HELLO passes against a peer with the - same key and fails against one still on the configured key. */ - uchar third_secret[ 32 ]; fd_memset( third_secret, 0x66, 32UL ); - uchar third_public[ 32 ]; - fd_ed25519_public_from_private( third_public, third_secret, sha ); + /* A new target must be reflected in both admission and HELLO. */ operator_frame( ctx, 4UL, 0x11, 0x77 ); - fd_memcpy( ctx->hello.staked_pubkey, third_public, 32UL ); - fd_memcpy( ctx->channel->self_hello.staked_pubkey, third_public, 32UL ); - fd_ed25519_sign( cert, msg, sizeof(msg), third_public, third_secret, sha ); - member_cert_signed( ctx, cert ); - FD_TEST( ctx->channel->member_cert_set ); + uchar public[ 32 ]; fd_memset( public, 0x5A, 32UL ); fd_failover_hello_t peer = ctx->channel->self_hello; peer.role = FD_FAILOVER_ROLE_ACTIVE; peer.boot_id = 77UL; @@ -629,10 +605,8 @@ test_dial_backoff_reset( void ) { controller_fini( ctx ); } -/* test_listen_slot: a full table makes room for a newcomer the listener - admitted. The address with the most candidates loses its oldest, - else the oldest of all goes. An authenticated, dialed or paired - candidate is never closed. */ +/* A full table evicts the oldest eligible candidate from the busiest + address. Reserved slots and candidates within their grace stay. */ static void fill( fd_failover_channel_t * ch, ulong idx, @@ -653,18 +627,15 @@ static void test_listen_slot( void ) { fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); fd_failover_channel_t * ch = ctx->channel; - FD_TEST( listen_slot( ch, 1000L )==0UL ); - for( ulong i=0UL; icandidates[ 3 ].fd==-1 ); - fill( ch, 3UL, FD_IP4_ADDR(10,2,0,1), 200L ); - FD_TEST( listen_slot( ch, 1000L )==0UL ); - fill( ch, 0UL, FD_IP4_ADDR(10,2,0,2), 201L ); - ch->candidates[ 1 ].phase = PHASE_READY; - ch->candidates[ 2 ].dialed = 1; - ch->paired_idx = 4; - FD_TEST( listen_slot( ch, 1000L )==5UL ); - ch->paired_idx = -1; + long now = FD_FAILOVER_ADMISSION_START_NANOS; + FD_TEST( listen_slot( ch, now )==UNAUTHENTICATED_IDX ); + for( ulong i=0UL; icandidates[ UNAUTHENTICATED_IDX ].fd ); + ch->candidates[ UNAUTHENTICATED_IDX ].fd = -1; + FD_TEST( listen_slot( ch, now )==UNAUTHENTICATED_IDX ); + FD_TEST( ch->candidates[ OUTBOUND_IDX ].fd!=-1 && ch->candidates[ ADMITTED_IDX ].fd!=-1 && + ch->candidates[ RECONNECT_IDX ].fd!=-1 ); for( ulong i=0UL; icandidates[ i ].fd!=-1 ) { close( ch->candidates[ i ].fd ); ch->candidates[ i ].fd = -1; From ba9e2c5c1155ae7b32dd2a9114dc62617b51b554 Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Tue, 20 Oct 2026 22:51:18 +0000 Subject: [PATCH 14/23] Generate failover junk identities in memory --- book/api/cli.md | 7 +- src/app/firedancer/config/default.toml | 2 +- src/app/firedancer/topology.c | 6 +- src/app/shared/commands/run/run.c | 130 +++++++--------------- src/app/shared/commands/run/run1.c | 4 + src/app/shared/fd_config.c | 8 -- src/app/shared/fd_config.h | 1 - src/app/shared/fd_config_json.c | 2 + src/disco/keyguard/fd_keyload.c | 64 +++++++++-- src/disco/keyguard/fd_keyload.h | 22 +++- src/disco/topo/fd_topo.h | 2 + src/disco/topo/fd_topo_run.c | 3 + src/discof/failover/fd_failover_channel.c | 6 +- src/discof/failover/fd_failover_tile.c | 38 ++++--- 14 files changed, 163 insertions(+), 132 deletions(-) diff --git a/book/api/cli.md b/book/api/cli.md index 77b12c61f20..086078a3cdb 100644 --- a/book/api/cli.md +++ b/book/api/cli.md @@ -399,9 +399,10 @@ cannot be authenticated, the local request cannot complete the handoff. Set `[failover.enabled]` to `true` on both validators. Configure the same staked identity in `[paths.identity_key]` and the same vote account in -`[paths.vote_account]`. Firedancer creates a junk identity for each -validator if one does not already exist. Both boot under their junk -identities without voting and listen for handoff requests. +`[paths.vote_account]`. Firedancer generates a fresh junk identity in memory +on each startup. This junk identity's key is not written to a file. Both +validators boot under their junk identities without voting and listen for +handoff requests. Once one validator is active, run the following on the standby to request a handoff. The staked identity keypair is already loaded from the diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index 4971afac388..5304f99e31d 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -1992,7 +1992,7 @@ telemetry = true # does not run failover, such as Agave. [failover] # Whether to accept incoming handoff requests. When true, the - # validator boots as standby under its junk identity. + # validator boots as standby under a junk identity generated in memory. # When false, it boots normally under [paths.identity_key]. For a # nonvoting requester, use an unstaked junk key as [paths.identity_key] # and the active's vote account as [paths.vote_account]. Request an diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index 05ae973b7e0..9fb8642316a 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -1,4 +1,5 @@ #include "topology.h" +#include "../../disco/keyguard/fd_keyload.h" #include "../shared/fd_config_json.h" #include "../../discof/poh/fd_poh.h" @@ -1431,8 +1432,8 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, set-identity installs a voting key. A demoted machine rebooting under the staked key would make the doppelganger rule shut down the promoted peer instead. */ - char const * identity_key_path = config->firedancer.failover.enabled - ? config->firedancer.failover.junk_identity_path + char const * identity_key_path = config->firedancer.failover.enabled && fd_topo_find_tile( &config->topo, "failov", 0UL )!=ULONG_MAX + ? FD_KEYLOAD_FAILOVER_JUNK : config->paths.identity_key; if( FD_UNLIKELY( !strcmp( tile->name, "metric" ) ) ) { @@ -1854,6 +1855,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) ) ) { fd_cstr_ncpy( tile->sign.identity_key_path, identity_key_path, sizeof(tile->sign.identity_key_path) ); + tile->sign.identity_key_fd = -1; tile->sign.failover_enabled = fd_topo_find_tile( &config->topo, "failov", 0UL )!=ULONG_MAX; fd_cstr_ncpy( tile->sign.failover_staked_key_path, config->firedancer.failover.enabled ? config->paths.identity_key : "", sizeof(tile->sign.failover_staked_key_path) ); diff --git a/src/app/shared/commands/run/run.c b/src/app/shared/commands/run/run.c index 73b61f52a4c..0ad2f84ab18 100644 --- a/src/app/shared/commands/run/run.c +++ b/src/app/shared/commands/run/run.c @@ -27,7 +27,7 @@ #include "../../../../disco/sleep/fd_sleep.h" #include "../../../../util/pod/fd_pod_format.h" #include "../../../../discof/restore/utils/fd_wfs.h" -#include "../../../../ballet/ed25519/fd_ed25519.h" +#include "../../../../disco/keyguard/fd_keyload.h" #include "../configure/configure.h" #include "../configure/fd_cpu_isolation.h" @@ -42,7 +42,6 @@ #include #include #include -#include #include #include #include @@ -279,6 +278,11 @@ execve_tile( char const * name, pid_t child = fork(); if( FD_UNLIKELY( -1==child ) ) FD_LOG_ERR(( "fork() failed (%i-%s)", errno, fd_io_strerror( errno ) )); if( FD_LIKELY( !child ) ) { + if( !strcmp( tile->name, "sign" ) && !strcmp( tile->sign.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) { + if( FD_UNLIKELY( -1==fcntl( tile->sign.identity_key_fd, F_SETFD, 0 ) ) ) + FD_LOG_ERR(( "passing the boot identity pipe failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } + char _current_executable_path[ PATH_MAX ]; FD_TEST( -1!=fd_file_util_self_exe( _current_executable_path ) ); @@ -541,6 +545,12 @@ main_pid_namespace( void * _args ) { if( FD_UNLIKELY( fd_cpuset_setaffinity( 0, floating_cpu_set ) ) ) FD_LOG_ERR(( "fd_cpuset_setaffinity failed (%i-%s)", errno, fd_io_strerror( errno ) )); + for( ulong i=0UL; itopo.tile_cnt; i++ ) { + fd_topo_tile_t const * tile = &config->topo.tiles[ i ]; + if( strcmp( tile->name, "sign" ) || strcmp( tile->sign.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) continue; + if( FD_UNLIKELY( close( tile->sign.identity_key_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } + if( FD_UNLIKELY( close( config_memfd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); if( need_xdp ) { for( uint i=0U; itopo.tile_cnt; i++ ) { + fd_topo_tile_t const * tile = &config->topo.tiles[ i ]; + if( strcmp( tile->name, "sign" ) || strcmp( tile->sign.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) continue; + if( FD_UNLIKELY( close( tile->sign.identity_key_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } if( FD_UNLIKELY( close( pipefd[ 1 ] ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); @@ -1070,97 +1085,29 @@ fdctl_check_configure( config_t const * config ) { fd_cpu_isolation_check_cgroup( config ); } -/* Writes a new keypair to tmp and syncs it. */ +/* Each Sign tile gets its own one-shot pipe. Only the public key goes + into the config shared with other tiles and saved for CLI discovery. */ static void FD_FN_SENSITIVE -failover_junk_write( char const * tmp ) { - uchar keypair[ 64 ]; - for( long got=0L; got<32L; ) { - long n = getrandom( keypair+got, (ulong)(32L-got), GRND_RANDOM ); - if( FD_UNLIKELY( -1==n ) ) { - if( FD_LIKELY( errno==EINTR ) ) continue; - FD_LOG_ERR(( "could not create the failover junk identity, getrandom() failed (%i-%s)", errno, fd_io_strerror( errno ) )); - } - got += n; - } - fd_sha512_t sha[ 1 ]; - fd_ed25519_public_from_private( keypair+32UL, keypair, fd_sha512_join( fd_sha512_new( sha ) ) ); - - /* The JSON array of the 64 bytes, as `keys new` writes it. */ - char json[ 1UL+64UL*4UL+1UL ]; - ulong len = 0UL; - for( ulong i=0UL; i<64UL; i++ ) { - ulong n; - FD_TEST( fd_cstr_printf_check( json+len, sizeof(json)-len, &n, "%s%u", i ? "," : "[", (uint)keypair[ i ] ) ); - len += n; - } - json[ len++ ] = ']'; - fd_memzero_explicit( keypair, sizeof(keypair) ); - - int fd = open( tmp, O_WRONLY|O_CREAT|O_EXCL|O_NOFOLLOW|O_CLOEXEC, S_IRUSR|S_IWUSR ); - if( FD_UNLIKELY( -1==fd ) ) FD_LOG_ERR(( "could not create `%s` (%i-%s)", tmp, errno, fd_io_strerror( errno ) )); - for( ulong written=0UL; writtenfiredancer.failover.junk_identity_path; - char tmp[ PATH_MAX ]; - char dir[ PATH_MAX ]; - if( FD_UNLIKELY( !fd_cstr_printf_check( tmp, sizeof(tmp), NULL, "%s.tmp", path ) ) ) FD_LOG_ERR(( "failover junk identity path `%s` is too long", path )); - fd_cstr_ncpy( dir, path, sizeof(dir) ); - char * slash = strrchr( dir, '/' ); - if( FD_UNLIKELY( !slash ) ) FD_LOG_ERR(( "failover junk identity path `%s` has no directory", path )); - *slash = '\0'; - - gid_t gid = getegid(); - uid_t uid = geteuid(); - if( FD_LIKELY( !gid && setegid( config->gid ) ) ) FD_LOG_ERR(( "setegid() failed (%i-%s)", errno, fd_io_strerror( errno ) )); - if( FD_LIKELY( !uid && seteuid( config->uid ) ) ) FD_LOG_ERR(( "seteuid() failed (%i-%s)", errno, fd_io_strerror( errno ) )); - - /* A crash can leave the temporary copy, a whole keypair, behind. */ - if( FD_UNLIKELY( unlink( tmp ) && errno!=ENOENT ) ) FD_LOG_ERR(( "could not remove `%s` (%i-%s)", tmp, errno, fd_io_strerror( errno ) )); - - int fd = open( path, O_RDONLY|O_NOFOLLOW|O_NONBLOCK|O_CLOEXEC ); - if( FD_UNLIKELY( -1==fd && errno==ENOENT ) ) { - if( FD_UNLIKELY( -1==fd_file_util_mkdir_all( dir, config->uid, config->gid, 1 ) ) ) FD_LOG_ERR(( "could not create `%s` (%i-%s)", dir, errno, fd_io_strerror( errno ) )); - failover_junk_write( tmp ); - if( FD_UNLIKELY( rename( tmp, path ) ) ) FD_LOG_ERR(( "could not rename `%s` to `%s` (%i-%s)", tmp, path, errno, fd_io_strerror( errno ) )); - int dir_fd = open( dir, O_RDONLY|O_DIRECTORY|O_CLOEXEC ); - if( FD_UNLIKELY( -1==dir_fd || fsync( dir_fd ) || close( dir_fd ) ) ) FD_LOG_ERR(( "could not sync `%s` (%i-%s)", dir, errno, fd_io_strerror( errno ) )); - FD_LOG_NOTICE(( "created the failover junk identity `%s`, it is this machine's own unique key, never copy it", path )); - } else { - struct stat st; - if( FD_UNLIKELY( -1==fd ) ) FD_LOG_ERR(( "could not open the failover junk identity `%s` (%i-%s), it has to be a regular file", path, errno, fd_io_strerror( errno ) )); - if( FD_UNLIKELY( fstat( fd, &st ) ) ) FD_LOG_ERR(( "could not stat the failover junk identity `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) )); - if( FD_UNLIKELY( close( fd ) ) ) FD_LOG_ERR(( "could not close the failover junk identity `%s` (%i-%s)", path, errno, fd_io_strerror( errno ) )); - /* Use the same size bounds as the key loader. */ - if( FD_UNLIKELY( !S_ISREG( st.st_mode ) || st.st_size<129L || st.st_size>1023L ) ) { - FD_LOG_ERR(( "the failover junk identity `%s` is not a keypair file, it is generated, so delete it and restart to make a new one, never put the staked key there", path )); - } - if( FD_UNLIKELY( st.st_uid!=config->uid && st.st_uid ) ) { - FD_LOG_ERR(( "the failover junk identity `%s` is owned by uid %u, not by the validator user or root", path, (uint)st.st_uid )); - } - if( FD_UNLIKELY( st.st_mode & (S_IRWXG|S_IRWXO) ) ) { - FD_LOG_ERR(( "the failover junk identity `%s` can be read by other users, run `chmod 600` on it", path )); +failover_junk_prepare( config_t * config ) { + if( fd_topo_find_tile( &config->topo, "failov", 0UL )==ULONG_MAX ) return; + uchar const * key = fd_keyload_generate(); + fd_memcpy( config->topo.boot_identity, key+32UL, 32UL ); + for( ulong i=0UL; itopo.tile_cnt; i++ ) { + fd_topo_tile_t * tile = &config->topo.tiles[ i ]; + if( strcmp( tile->name, "sign" ) ) continue; + FD_TEST( !strcmp( tile->sign.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ); + int pipefd[ 2 ]; + if( FD_UNLIKELY( pipe2( pipefd, O_CLOEXEC ) ) ) FD_LOG_ERR(( "pipe2() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + for( ulong off=0UL; off<64UL; ) { + long n = write( pipefd[ 1 ], key+off, 64UL-off ); + if( FD_UNLIKELY( n==-1L && errno==EINTR ) ) continue; + if( FD_UNLIKELY( n<=0L ) ) FD_LOG_ERR(( "could not deliver the generated boot identity" )); + off += (ulong)n; } + if( FD_UNLIKELY( close( pipefd[ 1 ] ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + tile->sign.identity_key_fd = pipefd[ 0 ]; } - - if( FD_UNLIKELY( seteuid( uid ) ) ) FD_LOG_ERR(( "seteuid() failed (%i-%s)", errno, fd_io_strerror( errno ) )); - if( FD_UNLIKELY( setegid( gid ) ) ) FD_LOG_ERR(( "setegid() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + fd_keyload_unload( key, 0 ); } void @@ -1172,8 +1119,7 @@ run_firedancer_init( config_t * config, if( FD_UNLIKELY( -1==err && errno==ENOENT ) ) FD_LOG_ERR(( "[consensus.identity_path] key does not exist `%s`. You can generate an identity key at this path by running `%s keys new %s --config `", config->paths.identity_key, FD_BINARY_NAME, config->paths.identity_key )); else if( FD_UNLIKELY( -1==err ) ) FD_LOG_ERR(( "could not stat [consensus.identity_path] `%s` (%i-%s)", config->paths.identity_key, errno, fd_io_strerror( errno ) )); - /* Failover boots under a junk identity of this machine's own, created - on the first boot. */ + /* Failover boots under a fresh identity kept only in memory. */ if( FD_UNLIKELY( config->firedancer.failover.enabled ) ) failover_junk_prepare( config ); /* FIXME: fdctl_check_configure unconditionally checks for network diff --git a/src/app/shared/commands/run/run1.c b/src/app/shared/commands/run/run1.c index 43b4cde9d20..ad0a52ccc6b 100644 --- a/src/app/shared/commands/run/run1.c +++ b/src/app/shared/commands/run/run1.c @@ -2,6 +2,7 @@ #include "run.h" #include "../../../../util/tile/fd_tile_private.h" +#include "../../../../disco/keyguard/fd_keyload.h" #include #include /* strtoul */ @@ -93,6 +94,9 @@ run1_cmd_fn( args_t * args, int flags = config->development.sandbox ? CLONE_NEWPID : 0; pid_t clone_pid = clone( tile_main, (uchar *)stack + FD_TILE_PRIVATE_STACK_SZ, flags, &clone_args ); if( FD_UNLIKELY( clone_pid<0 ) ) FD_LOG_ERR(( "clone() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( !strcmp( tile->name, "sign" ) && !strcmp( tile->sign.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) { + if( FD_UNLIKELY( close( tile->sign.identity_key_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + } if( FD_LIKELY( args->run1.pipe_fd!=-1 ) ) { ulong pid = (ulong)clone_pid; diff --git a/src/app/shared/fd_config.c b/src/app/shared/fd_config.c index 49089525e4b..9685723219d 100644 --- a/src/app/shared/fd_config.c +++ b/src/app/shared/fd_config.c @@ -181,14 +181,6 @@ fd_config_fillf( fd_config_t * config ) { replace( config->firedancer.paths.authorized_voter_paths[ i ], "{user}", config->user ); replace( config->firedancer.paths.authorized_voter_paths[ i ], "{name}", config->name ); } - - if( FD_UNLIKELY( config->firedancer.failover.enabled ) ) { - FD_TEST( fd_cstr_printf_check( config->firedancer.failover.junk_identity_path, - sizeof(config->firedancer.failover.junk_identity_path), - NULL, - "%s/junk-identity.json", - config->paths.base ) ); - } } static void diff --git a/src/app/shared/fd_config.h b/src/app/shared/fd_config.h index 0f20870a307..d46b9e6c366 100644 --- a/src/app/shared/fd_config.h +++ b/src/app/shared/fd_config.h @@ -130,7 +130,6 @@ struct fd_configf { int enabled; char listen_address[ 16 ]; ushort listen_port; - char junk_identity_path[ PATH_MAX ]; /* derived, [paths.base]/junk-identity.json when enabled */ } failover; }; diff --git a/src/app/shared/fd_config_json.c b/src/app/shared/fd_config_json.c index df8430a13a6..f1259e33400 100644 --- a/src/app/shared/fd_config_json.c +++ b/src/app/shared/fd_config_json.c @@ -12,6 +12,8 @@ or knowingly skipped) before the constant is bumped. String keys of the user's own file are separately forced through the classification lists below. */ +/* topo.boot_identity and sign.identity_key_fd are derived startup state, + not operator configuration. */ FD_STATIC_ASSERT( sizeof(fd_config_t)==24458368UL, update_fd_config_to_json_for_the_layout_change ); #define REDACTED "[redacted]" diff --git a/src/disco/keyguard/fd_keyload.c b/src/disco/keyguard/fd_keyload.c index b2add83b7f1..2a9cd05fc3c 100644 --- a/src/disco/keyguard/fd_keyload.c +++ b/src/disco/keyguard/fd_keyload.c @@ -1,5 +1,6 @@ #define _GNU_SOURCE #include "fd_keyload.h" +#include "../../ballet/ed25519/fd_ed25519.h" #include #include @@ -8,6 +9,17 @@ #include #include #include +#include + +static FD_TL uchar boot_public_key[ 32 ]; +static FD_TL int boot_key_fd = -1; + +void +fd_keyload_set_boot_identity( uchar const public_key[ 32 ], + int key_fd ) { + fd_memcpy( boot_public_key, public_key, 32UL ); + boot_key_fd = key_fd; +} uchar * FD_FN_SENSITIVE fd_keyload_read( int key_fd, @@ -79,6 +91,32 @@ read_key( char const * key_path, return fd_keyload_read( key_fd, key_path, key ); } +static void FD_FN_SENSITIVE +read_boot_key( uchar * key ) { + int key_fd = boot_key_fd; + boot_key_fd = -1; + struct stat st; + if( FD_UNLIKELY( fstat( key_fd, &st ) || !S_ISFIFO( st.st_mode ) ) ) + FD_LOG_ERR(( "the generated boot identity requires a startup pipe" )); + for( ulong off=0UL; off<64UL; ) { + long n = read( key_fd, key+off, 64UL-off ); + if( FD_UNLIKELY( n==-1L && errno==EINTR ) ) continue; + if( FD_UNLIKELY( n<=0L ) ) FD_LOG_ERR(( "could not read the generated boot identity" )); + off += (ulong)n; + } + long n; + do n = read( key_fd, key+64UL, 1UL ); while( FD_UNLIKELY( n==-1L && errno==EINTR ) ); + if( FD_UNLIKELY( n ) ) FD_LOG_ERR(( "unexpected data after the generated boot identity" )); + if( FD_UNLIKELY( close( key_fd ) ) ) FD_LOG_ERR(( "closing the boot identity pipe failed (%i-%s)", errno, fd_io_strerror( errno ) )); + fd_sha512_t * sha = fd_sha512_join( fd_sha512_new( key+FD_SHA512_ALIGN ) ); + FD_TEST( sha ); + fd_ed25519_public_from_private( key+64UL, key, sha ); + if( FD_UNLIKELY( !fd_memeq( key+32UL, key+64UL, 32UL ) || + !fd_memeq( key+32UL, boot_public_key, 32UL ) ) ) + FD_LOG_ERR(( "the generated boot identity does not match its public key" )); + fd_memzero_explicit( key+64UL, 4096UL-64UL ); +} + uchar const * FD_FN_SENSITIVE fd_keyload_load( char const * key_path, int public_key_only ) { @@ -91,16 +129,28 @@ fd_keyload_load( char const * key_path, pages. */ uchar * key_page = fd_keyload_alloc_protected_pages( 1UL, 2UL ); - read_key( key_path, key_page ); + if( FD_UNLIKELY( !strcmp( key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) ) { + if( FD_UNLIKELY( fd_memeq( boot_public_key, (uchar[ 32 ]){0}, 32UL ) ) ) FD_LOG_ERR(( "missing generated boot identity" )); + if( public_key_only ) fd_memcpy( key_page+32UL, boot_public_key, 32UL ); + else read_boot_key( key_page ); + } else { + read_key( key_path, key_page ); + } if( public_key_only ) fd_memzero_explicit( key_page, 32UL ); - /* Make the key page read-only now that the key has been loaded. */ - if( FD_UNLIKELY( mprotect( key_page, 4096UL, PROT_READ ) ) ) - FD_LOG_ERR(( "mprotect failed (%i-%s)", errno, fd_io_strerror( errno ) )); + return fd_keyload_mprotect_ro( public_key_only ? key_page+32UL : key_page, public_key_only ); +} - if( public_key_only ) return key_page+32UL; - else return key_page; +uchar const * FD_FN_SENSITIVE +fd_keyload_generate( void ) { + uchar * key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); + if( FD_UNLIKELY( !fd_rng_secure( key, 32UL ) ) ) FD_LOG_ERR(( "could not generate the boot identity" )); + fd_sha512_t * sha = fd_sha512_join( fd_sha512_new( key+FD_SHA512_ALIGN ) ); + FD_TEST( sha ); + fd_ed25519_public_from_private( key+32UL, key, sha ); + fd_memzero_explicit( key+FD_SHA512_ALIGN, sizeof(fd_sha512_t) ); + return fd_keyload_mprotect_ro( key, 0 ); } uchar * FD_FN_SENSITIVE @@ -126,7 +176,7 @@ void FD_FN_SENSITIVE fd_keyload_unload( uchar const * key, int public_key_only ) { void * key_page = public_key_only ? (uchar *)key-32UL : (uchar *)key; - ulong sz = (2UL*1UL+2UL)*4096UL; + ulong sz = (2UL*2UL+1UL)*4096UL; if( FD_UNLIKELY( mprotect( key_page, 4096UL, PROT_READ | PROT_WRITE ) ) ) FD_LOG_ERR(( "mprotect failed (%i-%s)", errno, fd_io_strerror( errno ) )); diff --git a/src/disco/keyguard/fd_keyload.h b/src/disco/keyguard/fd_keyload.h index aef02816acc..c443679895b 100644 --- a/src/disco/keyguard/fd_keyload.h +++ b/src/disco/keyguard/fd_keyload.h @@ -3,6 +3,9 @@ #include "../fd_disco_base.h" +/* Internal identity source selected by failover startup. */ +#define FD_KEYLOAD_FAILOVER_JUNK "@failover-junk" + /* fd_keyload_read() reads a JSON encoded keypair from the provided file descriptor. The key_path is not opened or read from, it is only used to output diagnostic error messages if reading the key fails. @@ -30,7 +33,9 @@ fd_keyload_read( int key_fd, key_path must point to the first letter in a NUL-terminated cstr that is the path on disk of the key file. The key file must exist, be readable, and have the form of a Solana keypair (64 element JSON - array of bytes). If public_key_only is non-zero, zeros out the + array of bytes). FD_KEYLOAD_FAILOVER_JUNK instead selects the + generated identity installed by fd_keyload_set_boot_identity. + If public_key_only is non-zero, zeros out the private part of the key and returns a pointer to the first byte (of 32) of the public part of the key in binary format. If public_key_only is zero, returns a pointer to the first byte (of 64) @@ -51,6 +56,21 @@ uchar const * FD_FN_SENSITIVE fd_keyload_load( char const * key_path, int public_key_only ); +/* Set this tile thread's generated boot public key and private startup + pipe before loading FD_KEYLOAD_FAILOVER_JUNK. Only Sign gets a pipe, + other tiles pass -1 and can load only the public key. */ + +void +fd_keyload_set_boot_identity( uchar const public_key[ 32 ], + int key_fd ); + +/* Generate a fresh Ed25519 keypair in protected memory. The returned + read-only key has the same unload contract as fd_keyload_load with + public_key_only clear. No key material is written to a file. */ + +uchar const * FD_FN_SENSITIVE +fd_keyload_generate( void ); + /* fd_keyload_mprotect_wr() makes a key page loaded with fd_keyload_load writable. Accepts the const pointer returned by fd_keyload_load and returns a non-const pointer that the caller may diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index 22ebd907c0b..05bad95648c 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -414,6 +414,7 @@ struct fd_topo_tile { struct { char identity_key_path[ PATH_MAX ]; + int identity_key_fd; int failover_enabled; char failover_staked_key_path[ PATH_MAX ]; ulong authorized_voter_paths_cnt; @@ -824,6 +825,7 @@ typedef struct { between them. */ struct fd_topo { char app_name[ 256UL ]; + uchar boot_identity[ 32 ]; /* generated public key, zero for a file-backed identity */ uchar props[ 32768UL ]; ulong sleep_obj_id; diff --git a/src/disco/topo/fd_topo_run.c b/src/disco/topo/fd_topo_run.c index 08fbf80c7cd..22f83088ea9 100644 --- a/src/disco/topo/fd_topo_run.c +++ b/src/disco/topo/fd_topo_run.c @@ -6,6 +6,7 @@ #include "../sleep/fd_sleep.h" #include "../metrics/fd_metrics.h" #include "../events/fd_event_report.h" +#include "../keyguard/fd_keyload.h" #include "../../util/tile/fd_tile_private.h" #include @@ -78,6 +79,8 @@ fd_topo_run_tile( fd_topo_t * topo, fd_topo_join_tile_workspaces( topo, tile, core_dump_level ); fd_topo_mlock_code(); + fd_keyload_set_boot_identity( topo->boot_identity, strcmp( tile->name, "sign" ) ? -1 : tile->sign.identity_key_fd ); + if( FD_UNLIKELY( tile_run->privileged_init ) ) tile_run->privileged_init( topo, tile ); diff --git a/src/discof/failover/fd_failover_channel.c b/src/discof/failover/fd_failover_channel.c index 4b2fc61b051..7504f5ae1ba 100644 --- a/src/discof/failover/fd_failover_channel.c +++ b/src/discof/failover/fd_failover_channel.c @@ -621,9 +621,9 @@ hello_err_name( int err ) { case FD_FAILOVER_HELLO_ERR_VERSION: return "the protocol version differs, run the same Firedancer version on both machines"; case FD_FAILOVER_HELLO_ERR_STAKED: return "the failover identity differs, check that both machines run [paths.identity_key] with the same keypair, or the same key from their last `set-identity`"; case FD_FAILOVER_HELLO_ERR_VOTE_ACCT: return "the vote account differs, check that [paths.vote_account] is the same on both machines"; - case FD_FAILOVER_HELLO_ERR_JUNK_EQ: return "its junk key is ours, check that the address is the other machine, and if both machines share one [paths.base]/junk-identity.json delete it on one of them and restart that machine, never copy that file"; - case FD_FAILOVER_HELLO_ERR_JUNK_STAKE: return "a junk key is the staked key, [paths.base]/junk-identity.json must not be the staked [paths.identity_key]"; - case FD_FAILOVER_HELLO_ERR_BOTH_ACT: return "both members are active, only one machine may run the staked identity, stop signing now on the machine that should not vote with `set-identity` and its junk identity there"; + case FD_FAILOVER_HELLO_ERR_JUNK_EQ: return "its junk key is ours, check that the address is the other validator"; + case FD_FAILOVER_HELLO_ERR_JUNK_STAKE: return "a boot identity is the staked key, the standby must use a different identity"; + case FD_FAILOVER_HELLO_ERR_BOTH_ACT: return "both members are active, only one machine may run the staked identity, stop the validator that should not vote, a failover-enabled validator restarts as standby"; case FD_FAILOVER_HELLO_ERR_ROLE: return "the role is unknown, run the same Firedancer version on both machines"; case FD_FAILOVER_HELLO_ERR_BOOT_ID: return "the boot id is zero, run the same Firedancer version on both machines"; case FD_FAILOVER_HELLO_ERR_MODE: return "the consensus mode differs, run both machines in the same consensus mode"; diff --git a/src/discof/failover/fd_failover_tile.c b/src/discof/failover/fd_failover_tile.c index 4b904bca9ff..90fa00a8787 100644 --- a/src/discof/failover/fd_failover_tile.c +++ b/src/discof/failover/fd_failover_tile.c @@ -108,7 +108,7 @@ struct fd_failover_request { uint addr; /* the address we dial, a resume may replace it */ long until; /* when the request pauses, zero while paused */ int sent; /* sent on this session */ - int peer_restarted; /* paused because the boot we authenticated restarted */ + int peer_restarted; /* paused because the authenticated member or boot changed */ ulong last_id; /* id of our latest request, kept for status */ ulong result; /* FD_FAILOVER_HANDOFF_* of our latest request */ ulong send_cnt; /* how many times we sent it */ @@ -353,10 +353,8 @@ privileged_init( fd_topo_t const * topo, ctx->tower_seen_seq = ULONG_MAX; FD_TEST( fd_rng_secure( &ctx->handoff_base, sizeof(ulong) ) ); - /* The junk identity is this machine's own key, it goes into HELLO and - TLS. We keep only the public keys of it and of the staked - [paths.identity_key]. Like every tile's identity load, the loader - reads the whole file and zeroes the private half before it returns. */ + /* The boot identity goes into HELLO and TLS. With failover enabled + it is generated for this boot. Only Sign holds its private key. */ uchar const * junk_pubkey = fd_keyload_load( tile->failov.identity_key_path, 1 ); fd_memcpy( ctx->hello.junk_pubkey, junk_pubkey, 32UL ); fd_keyload_unload( junk_pubkey, 1 ); @@ -369,7 +367,7 @@ privileged_init( fd_topo_t const * topo, } if( FD_UNLIKELY( fd_memeq( ctx->hello.junk_pubkey, ctx->hello.staked_pubkey, 32UL ) ) ) { - FD_LOG_ERR(( "the failover junk identity `%s` is the staked [paths.identity_key], each failover machine boots under its own key", tile->failov.identity_key_path )); + FD_LOG_ERR(( "the boot identity must differ from the staked [paths.identity_key]" )); } /* The vote key is a base58 pubkey or a keypair file, same as in the @@ -406,7 +404,7 @@ privileged_init( fd_topo_t const * topo, FD_TEST( ctx->channel ); fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 }; if( FD_UNLIKELY( !ctx->target_pending && fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, (fd_failover_proof_signer_t){ .ctx=ctx, .sign_fn=sign_proof }, &ctx->hello ) ) ) { - FD_LOG_ERR(( "failover TLS setup failed for the junk identity `%s`", tile->failov.identity_key_path )); + FD_LOG_ERR(( "failover TLS setup failed for the boot identity" )); } if( !ctx->client_only ) fd_failover_channel_init_listener( ctx->channel, tile->failov.listen_addr, tile->failov.port ); @@ -1651,7 +1649,10 @@ request_wait( fd_failover_tile_ctx_t const * ctx ) { /* What the operator can do about a paused request. */ static char const * request_next( fd_failover_tile_ctx_t const * ctx ) { - if( FD_UNLIKELY( ctx->request.peer_restarted ) ) return "the active boot we authenticated restarted, `failover promote` here starts a new handoff"; + if( FD_UNLIKELY( ctx->request.peer_restarted ) ) { + if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) return "the authenticated peer changed, `failover promote` here ends the confirmation wait without changing our identity"; + return "the authenticated member or boot changed, `failover promote` here starts a new handoff"; + } switch( ctx->action ) { case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER: if( !ctx->request.send_cnt ) return "nothing was sent and the active is unchanged, `failover promote` here resumes the request, " @@ -2001,10 +2002,11 @@ peer_poll( fd_failover_tile_ctx_t * ctx, request_end( ctx, now, active ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ); return; } - /* A changed peer pauses the request without replacing its binding. - The same member on a new boot will not demote for it. */ - ctx->request.peer_restarted = ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && fd_memeq( peer->junk_pubkey, ctx->request.peer.junk, 32UL ); - request_pause( ctx, now, ctx->request.peer_restarted ? "the authenticated boot restarted" : "authenticated member or boot changed" ); + /* A fresh boot has a new junk key. Keep the old binding until the + operator ends this request, without accepting another peer's final + state or acknowledgement for it. */ + ctx->request.peer_restarted = 1; + request_pause( ctx, now, "authenticated member or boot changed" ); return; } fd_failover_handoff_request_t request = { .handoff_id=ctx->request.id, .target_boot_id=ctx->request.peer.boot_id, .replay_slot=ctx->replay_slot }; @@ -2085,8 +2087,16 @@ apply_control( fd_failover_tile_ctx_t * ctx, case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: { int paused = request_paused( ctx ) && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT; if( FD_UNLIKELY( paused && ctx->request.peer_restarted ) ) { - /* The active restarted before it demoted, so start a new request. */ - FD_LOG_NOTICE(( "handoff %lu was bound to active boot %016lx, which restarted, ending it and starting a new handoff", ctx->request.id, ctx->request.peer.boot_id )); + if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) { + /* Local adoption is already settled. This command ends only + our wait, without confirming anything on the old peer. */ + int active = ctx->role==FD_FAILOVER_ROLE_ACTIVE; + FD_LOG_WARNING(( "handoff %lu: the authenticated peer changed before it confirmed our answer, ending the confirmation wait, %s, the old peer's acknowledgement remains unknown", + ctx->request.id, active ? "we hold the staked identity" : "we remain standby" )); + request_end( ctx, now, active ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ); + return FD_ADMINCTL_RESULT_SUCCESS; + } + FD_LOG_NOTICE(( "handoff %lu was bound to active boot %016lx, the authenticated peer changed, ending it and starting a new handoff", ctx->request.id, ctx->request.peer.boot_id )); request_end( ctx, now, FD_FAILOVER_HANDOFF_RESTARTED ); paused = 0; } From 3cfea9aa26a1d4b68c528807315fd6ffded41550 Mon Sep 17 00:00:00 2001 From: cmoyes-jump Date: Wed, 21 Oct 2026 10:26:58 -0500 Subject: [PATCH 15/23] failover: make handoffs Alpenglow-only --- book/api/cli.md | 51 +- book/api/metrics-generated.md | 24 - book/snippets/commands/failover-status.ansi | 2 +- src/app/firedancer/commands/failover.c | 29 +- src/app/firedancer/config/default.toml | 6 +- src/app/firedancer/topology.c | 26 +- src/app/shared/fd_config.c | 5 - src/app/shared/test_config_parse.c | 3 +- src/choreo/votor/Local.mk | 8 +- src/choreo/votor/ag_hist.c | 87 +++ src/choreo/votor/ag_hist.h | 75 ++ src/choreo/votor/ag_votor.c | 157 +++++ src/choreo/votor/ag_votor.h | 35 + src/choreo/votor/test_ag_hist.c | 25 + src/choreo/votor/test_ag_votor_failover.c | 293 ++++++++ src/disco/keyguard/fd_keyswitch.h | 2 +- src/disco/metrics/generate/types.py | 1 - src/disco/metrics/generated/fd_metrics_all.c | 21 - src/disco/metrics/generated/fd_metrics_all.h | 3 +- .../metrics/generated/fd_metrics_enums.h | 10 - .../metrics/generated/fd_metrics_failov.h | 97 --- .../metrics/generated/fd_metrics_sample_gen.c | 2 +- src/disco/metrics/metrics.xml | 22 - src/disco/stem/fd_stem.c | 7 +- src/disco/stem/fd_stem_run1.c | 3 - src/disco/topo/fd_topo.h | 3 +- src/discof/admin/fd_admin_tile.c | 45 +- src/discof/admin/fd_adminctl.h | 18 +- src/discof/admin/test_admin_tile.c | 8 +- src/discof/failover/Local.mk | 2 + src/discof/failover/fd_failover_bus.h | 6 +- src/discof/failover/fd_failover_channel.c | 23 +- src/discof/failover/fd_failover_channel.h | 21 - src/discof/failover/fd_failover_proto.c | 31 +- src/discof/failover/fd_failover_proto.h | 30 +- src/discof/failover/fd_failover_tile.c | 667 +++++++++--------- src/discof/failover/test_failover_channel.c | 2 +- src/discof/failover/test_failover_proto.c | 35 +- src/discof/failover/test_failover_tile.c | 71 +- src/discof/failover/test_failover_tile_ag.c | 302 ++++++++ src/discof/votor/fd_votor_tile.c | 357 +++++++++- src/discof/votor/fd_votor_tile.h | 38 + src/discof/votor/test_votor_tile.c | 338 +++++++++ 43 files changed, 2247 insertions(+), 744 deletions(-) create mode 100644 src/choreo/votor/ag_hist.c create mode 100644 src/choreo/votor/ag_hist.h create mode 100644 src/choreo/votor/test_ag_hist.c create mode 100644 src/choreo/votor/test_ag_votor_failover.c delete mode 100644 src/disco/metrics/generated/fd_metrics_failov.h create mode 100644 src/discof/failover/test_failover_tile_ag.c diff --git a/book/api/cli.md b/book/api/cli.md index 086078a3cdb..b2a48e2d22a 100644 --- a/book/api/cli.md +++ b/book/api/cli.md @@ -222,7 +222,7 @@ running as is refused. Between two Firedancer validators running failover, only use `firedancer failover promote`, which hands over the vote history directly. Do not use `--vote-history-file` for this, it is only for a -tower file from a validator that does not run failover, such as Agave. +vote history file from a validator that does not run failover, such as Agave. With failover on, `set-identity` also makes the new key the identity `failover promote` hands off, unless it is the machine's junk identity. @@ -230,10 +230,7 @@ Only a machine running the same key can take it over, so set that key as `[paths.identity_key]` on the other machine and restart it. After a restart, the machine boots as standby under its junk identity and uses `[paths.identity_key]` as the handoff target again. -The new key cannot also be one of the authorized voters. With -`--vote-history-file`, a failover machine does not vote until the -file's root has been replayed, and ignores a file whose root is not on -the fork of its own root, waiting out the lockouts of its votes. +The new key cannot also be one of the authorized voters. The validator writes `vote_history-.bin` in the `[paths.vote_history]` directory before each vote and each block it @@ -356,7 +353,8 @@ standby. The remote validator must have `[failover.enabled]` set to `true` to accept the request. Both validators must use the same vote account. -The failover commands are only supported by the full Firedancer client. +The failover commands require the full Firedancer client with +Alpenglow consensus enabled through `[consensus.alpenglow]` or `--alpenglow`. ```sh firedancer failover promote [] [--name ] [--yes] [--force] [--address
] [--port ] @@ -400,9 +398,8 @@ cannot be authenticated, the local request cannot complete the handoff. Set `[failover.enabled]` to `true` on both validators. Configure the same staked identity in `[paths.identity_key]` and the same vote account in `[paths.vote_account]`. Firedancer generates a fresh junk identity in memory -on each startup. This junk identity's key is not written to a file. Both -validators boot under their junk identities without voting and listen for -handoff requests. +on each startup. The private key is never written to a file. Both validators +boot under their junk identities without voting and listen for handoff requests. Once one validator is active, run the following on the standby to request a handoff. The staked identity keypair is already loaded from the @@ -445,7 +442,7 @@ firedancer keys new /path/to/standby-identity.json firedancer run --config /path/to/standby.toml ``` -Validators with a vote account include a failover tile for outgoing +Alpenglow validators with a vote account include a failover tile for outgoing requests, so allow for it when assigning CPU affinity. Pass the active's identity keypair on first use. This prepares the target without installing it. The requester must still be using the identity it started with. The @@ -472,11 +469,11 @@ The following applies to both configurations. The remote validator refuses the handoff if it has no final vote history to hand over, the local standby has not started replaying, or the standby -is more than 64 slots behind its last vote. It keeps its identity in -these cases. It also refuses a new request while another transition is -running. Once it agrees to hand off, it gives up the identity before the -local validator adopts its vote history. If adoption fails or stalls, -the handoff can fail with neither machine voting. +is more than 64 slots behind its finality anchor. It keeps its identity in +these cases. It also refuses a new request while another transition is running. Once it agrees to hand +off, it gives up the identity before the local validator adopts its +vote history. If adoption fails or stalls, the handoff can fail with +neither machine voting. Exit code 0 means the local validator accepted the request. It does not mean the remote validator has agreed to hand off or that the handoff has @@ -509,9 +506,9 @@ or `--port` when resuming it. With `--force`, the local standby takes the identity without contacting the remote validator. This is also needed for first use when both machines are standbys. Promotion first tries eligible saved peer vote -history, then the vote account, and starts with empty vote history if -neither can be adopted. Check for `role: active` and `action: idle` to -confirm completion. +history. If that history cannot be adopted, it falls back to empty vote +history with a slot bound that prevents voting at or below it. Check for +`role: active` and `action: idle` to confirm completion. Forced promotion can cancel a handoff that is waiting for the remote validator's response. It cannot interrupt a local vote history adoption @@ -521,8 +518,8 @@ identity can be installed. ::: warning WARNING Only use `--force` after ensuring every other machine with the identity -cannot sign. It skips the checks on the other machine and can discard -earlier vote lockouts by accepting incomplete or empty vote history. +cannot sign. It skips the checks on the other machine and can leave votes +and leader windows unaccounted for by accepting incomplete or empty history. Never force promotion on both machines at once. `--yes` only skips the confirmation prompt. It does not imply `--force` or suppress its warning. @@ -555,9 +552,9 @@ It exits with code 0 if status was retrieved, including when failover is disabled. Otherwise it exits with a non-zero code and prints the reason to `stderr`. -With `[failover.enabled]` set to `false` and a vote account configured, -status reports `client only, listener disabled` and shows outgoing -handoff progress. +With Alpenglow enabled, `[failover.enabled]` set to `false`, and a vote +account configured, status reports `client only, listener disabled` and +shows outgoing handoff progress. The following example shows a completed handoff. The last handoff ID matches the ID printed by `failover promote` above. The connection closes @@ -580,10 +577,10 @@ the highest slot known to have been signed locally or reported by the remote validator. Forced promotion can proceed without history covering that slot, as described above. -| Arguments | Description | -|-------------------|-------------| -| `--name ` | Select the validator instance to attach to if more than one is running on this host. | -| `--config ` | Optionally use the configuration TOML file the validator was started with. | +| Arguments | Description | +|-----------------------|-------------| +| `--name ` | Select the validator instance to attach to if more than one is running on this host. | +| `--config ` | Optionally use the configuration TOML file the validator was started with. | ## `remove-all-authorized-voters` Removes all authorized voters from the running validator, including any diff --git a/book/api/metrics-generated.md b/book/api/metrics-generated.md index 2996f107d99..94a47a3b506 100644 --- a/book/api/metrics-generated.md +++ b/book/api/metrics-generated.md @@ -1822,30 +1822,6 @@
-## Failov Tile - -
- -| Metric | Type | Description | -|--------|------|-------------| -| failov_​conn_​accepted | counter | TCP connections accepted before admission checks | -| failov_​accept_​budget_​exhausted | counter | Polls that stopped accepting because the syscall budget was empty | -| failov_​conn_​rejected
{failover_​conn_​reject_​reason="source"} | counter | Accepted TCP connections refused before candidate allocation, by reason (A candidate from this address is open or its admission quota has not expired) | -| failov_​conn_​rejected
{failover_​conn_​reject_​reason="budget"} | counter | Accepted TCP connections refused before candidate allocation, by reason (The ordinary or reconnect admission budget is empty) | -| failov_​conn_​rejected
{failover_​conn_​reject_​reason="capacity"} | counter | Accepted TCP connections refused before candidate allocation, by reason (The corresponding handshake pool is full) | -| failov_​conn_​reconnect_​admitted | counter | Candidates admitted to the reserved lane for recently authenticated addresses | -| failov_​handshake_​timed_​out | counter | Candidates closed at a phase or absolute handshake deadline | -| failov_​proof_​rejected | counter | Admission proofs that failed target identity authentication | -| failov_​hello_​rejected | counter | Admission or TLS HELLO checks that failed | -| failov_​tls_​handshake_​failed | counter | TLS handshakes that failed after admission | -| failov_​frame_​protocol_​rejected | counter | Authenticated frames rejected by the controller | -| failov_​conn_​authenticated | counter | Completed mutual admission, TLS and HELLO authentications | -| failov_​handshake_​duration_​nanos | counter | Cumulative time from candidate allocation to completed authentication | -| failov_​handshake_​latest_​duration_​nanos | gauge | Duration of the most recently completed authentication | -| failov_​conn_​pending | gauge | Candidates waiting for admission, TLS or HELLO authentication | - -
- ## Snapmk Tile
diff --git a/book/snippets/commands/failover-status.ansi b/book/snippets/commands/failover-status.ansi index 77b54553d0d..ab00e228fa7 100644 --- a/book/snippets/commands/failover-status.ansi +++ b/book/snippets/commands/failover-status.ansi @@ -9,5 +9,5 @@ peer boot id: 0000000000000001 peer address: 192.0.2.10:8010, from gossip last handoff: 123456789, taken other holder: n/a, this machine holds the identity -takeover adopts: the vote account (automatic fallback) +takeover adopts: an empty history (requires --force) coverage floor: slot 300 diff --git a/src/app/firedancer/commands/failover.c b/src/app/firedancer/commands/failover.c index c413f103f37..39b328a303a 100644 --- a/src/app/firedancer/commands/failover.c +++ b/src/app/firedancer/commands/failover.c @@ -18,13 +18,13 @@ static char const * const ACTION_NAMES[] = { "idle", "handing off, switching to the junk key", "handing off, waiting for the peer's answer", "promote, waiting for replay", "promote, waiting for vote history adoption", "promote, switching to the staked key", "handoff, requesting the active's final vote state", "handoff, waiting for completion", - "handing off, waiting for the tower to drain" + "handing off, waiting for the vote history stream to drain" }; static char const * const SESSION_NAMES[] = { "listening", "dialing", "hello", "paired", "backoff" }; static char const * const SOURCE_NAMES[] = { - "the stored peer tower", "the vote account (automatic fallback)" + "the stored peer vote history", "an empty history (requires --force)" }; static char const * const HANDOFF_NAMES[] = { "nothing handed over", "pending", "taken", "declined", "restarted", "cancelled by `failover promote --force`", @@ -71,7 +71,7 @@ failover_cmd_args( int * pargc, /* `failover promote --force` bypasses checks and takes the identity immediately without a clean handoff. */ args->failover.cmd = (int)( status ? FD_ADMINCTL_FAILOVER_CMD_STATUS : - args->failover.force ? FD_ADMINCTL_FAILOVER_CMD_PROMOTE : FD_ADMINCTL_FAILOVER_CMD_HANDOFF ); + args->failover.force ? FD_ADMINCTL_FAILOVER_CMD_PROMOTE : FD_ADMINCTL_FAILOVER_CMD_HANDOFF ); if( FD_UNLIKELY( args->failover.yes && args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_STATUS ) ) { FD_LOG_ERR(( "--yes is only meaningful for `failover promote`" )); @@ -150,8 +150,8 @@ control_result_name( ulong result ) { "finish it, if it shows `role: standby` nothing votes and `failover promote --force` on one machine takes the identity"; case FD_FAILOVER_CONTROL_RESULT_TAKEN: return "the peer took our last handoff and may still be voting"; case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "gossip showed the staked identity at another host within the last 15 seconds, an active is publishing"; - case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER: return "the active has no eligible final vote state to hand over and keeps the identity, check its voting progress and vote-history logs before retrying"; - case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: return "this standby is too far behind the active's last vote and the active keeps the identity, run `failover promote` again once this machine has caught up"; + case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: return "the active has no eligible final vote state to hand over and keeps the identity, check its voting progress and vote-history logs before retrying"; + case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: return "this standby is too far behind the active's vote history and the active keeps the identity, run `failover promote` again once this machine has caught up"; case FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED: return "provide the active's keypair with `failover promote ` to prepare this client's target identity"; case FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT: return "the keypair must match the prepared target and must differ from this machine's boot identity"; case FD_FAILOVER_CONTROL_RESULT_NOT_BOOT_IDENTITY: return "the client must still be using its boot identity when preparing a target"; @@ -176,16 +176,16 @@ failover_confirm( args_t const * args ) { "this machine is not aware of are not accounted for.\n" "WARNING: --force skips every check on the other machine. Use it only when that\n" "machine cannot sign, or both machines may vote with the staked identity.\n" - "It also permits incomplete or empty vote history, which may lose earlier lockouts.\n" - "Promotion tries eligible saved vote state first, then the vote account, and if the\n" - "vote account cannot be adopted it starts from an EMPTY vote history.\n" )); + "Promotion tries eligible saved vote history first, then starts from an EMPTY\n" + "history with a bound covering replay and known votes. Missing votes and leader\n" + "windows above that bound remain unprotected.\n" )); } if( FD_UNLIKELY( !args->failover.yes && isatty( STDIN_FILENO ) ) ) { if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) { FD_LOG_STDOUT(( "This asks the active to hand off the staked identity to this standby. The active\n" - "keeps it if this standby is more than 64 slots behind its last vote. Otherwise\n" - "it gives up the identity first, so if replay here stalls before it catches up\n" - "the handoff fails and neither machine votes until `failover promote --force`.\n" + "keeps it if this standby is more than 64 slots behind its finality anchor.\n" + "Otherwise, it gives up the identity before this validator adopts its history.\n" + "If replay stalls, the handoff can fail with neither machine voting.\n" "Check that this machine is caught up. Type yes to continue: " )); } else { FD_LOG_STDOUT(( "Type yes to continue: " )); @@ -229,7 +229,8 @@ failover_status_print( fd_adminctl_failover_status_resp_t const * resp ) { FD_LOG_STDOUT(( "%-22s %s\n", "failover:", resp->enabled==2 ? "client only, listener disabled" : "enabled" )); FD_LOG_STDOUT(( "%-22s %s\n", "role:", role_name ( resp->role ) )); if( FD_UNLIKELY( resp->request_paused ) ) { - FD_LOG_STDOUT(( "%-22s %s, paused at its deadline and not dialing, `failover promote` resumes it\n", "action:", action_name( resp->action ) )); + FD_LOG_STDOUT(( "%-22s %s, paused at its deadline and not dialing, `failover promote` resumes it\n", "action:", + action_name( resp->action ) )); } else { FD_LOG_STDOUT(( "%-22s %s\n", "action:", action_name( resp->action ) )); } @@ -306,7 +307,7 @@ failover_request( fd_adminctl_t * adminctl, static void __attribute__((noreturn)) fail_disabled( int status ) { if( status ) FD_LOG_ERR(( "`failover status` is unavailable because this validator has no failover command bus" )); - FD_LOG_ERR(( "the running validator has no failover client, use a supported Firedancer build with [paths.vote_account] configured" )); + FD_LOG_ERR(( "the running validator has no failover client, run Alpenglow consensus with [paths.vote_account] configured" )); } static void __attribute__((noreturn)) @@ -370,7 +371,7 @@ failover_cmd_fn( args_t * args, case FD_ADMINCTL_RESULT_SUCCESS: if( FD_UNLIKELY( resp_sz!=(status ? sizeof(resp.status) : sizeof(resp.control)) || (status ? resp.status.version : resp.control.version)!=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION || - (status && resp.status.mode>=FD_FAILOVER_MODE_CNT) ) ) { + (status && resp.status.mode!=FD_FAILOVER_MODE_ALPENGLOW) ) ) { fail_incompatible(); } if( status ) { diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index 5304f99e31d..89e060434f7 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -1991,8 +1991,10 @@ telemetry = true # history files are only for handing off an identity from a validator that # does not run failover, such as Agave. [failover] - # Whether to accept incoming handoff requests. When true, the - # validator boots as standby under a junk identity generated in memory. + # Whether to accept incoming handoff requests. Requires Alpenglow + # consensus, enabled by [consensus.alpenglow] or --alpenglow. When + # true, the validator boots as standby under a fresh junk identity generated + # in memory. Its private key is not written to a file. # When false, it boots normally under [paths.identity_key]. For a # nonvoting requester, use an unstaked junk key as [paths.identity_key] # and the active's vote account as [paths.vote_account]. Request an diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index 9fb8642316a..bef052ed03e 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -420,7 +420,8 @@ fd_topo_initialize( config_t * config ) { if( FD_UNLIKELY( failover_enabled ) ) { fd_topob_wksp( topo, "admin_failov" ); fd_topob_wksp( topo, "failov_admin" ); - if( !alpenglow_enabled ) fd_topob_wksp( topo, "adopt_tower" ); + fd_topob_wksp( topo, "failov_votor" ); + fd_topob_wksp( topo, "votor_hist" ); if( !config->tiles.gui.enabled ) fd_topob_wksp( topo, "gossip_ciseen" ); } @@ -528,10 +529,9 @@ fd_topo_initialize( config_t * config ) { if( FD_UNLIKELY( failover_enabled ) ) { /**/ fd_topob_link( topo, "admin_failov", "admin_failov", 32UL, FD_FAILOVER_BUS_MTU, 1UL ); /**/ fd_topob_link( topo, "failov_admin", "failov_admin", 32UL, FD_FAILOVER_BUS_MTU, 1UL ); - if( !alpenglow_enabled ) { - /**/ fd_topob_link( topo, "adopt_tower", "adopt_tower", 32UL, FD_FAILOVER_TOWER_STATE_MAX, 1UL ); - /**/ fd_topob_link( topo, "tower_adopt", "adopt_tower", 32UL, sizeof(fd_tower_adopt_result_t), 1UL ); - } + /**/ fd_topob_link( topo, "failov_votor", "failov_votor", 32UL, FD_FAILOVER_STATE_MAX, 1UL ); + /**/ fd_topob_link( topo, "votor_failov", "failov_votor", 32UL, sizeof(fd_votor_adopt_result_t), 1UL ); + /**/ fd_topob_link( topo, "votor_hist", "votor_hist", 1024UL, sizeof(fd_votor_hist_msg_t), 1UL ); if( !config->tiles.gui.enabled ) { fd_topob_link( topo, "gossip_ciseen", "gossip_ciseen", 65536UL*2UL, sizeof(fd_gossip_update_message_t), 1UL ); } @@ -792,10 +792,10 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_admin", 0UL ); /**/ fd_topob_tile_in ( topo, "admin", 0UL, "metric_in", "failov_admin", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); } - if( !alpenglow_enabled && failover_enabled ) { - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_out", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); - /**/ fd_topob_tile_out( topo, "failov", 0UL, "adopt_tower", 0UL ); - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "tower_adopt", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + if( FD_UNLIKELY( failover_enabled ) ) { + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_hist", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_votor", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); } FOR(execrp_tile_cnt) fd_topob_tile_in ( topo, "execrp", i, "metric_in", "replay_execrp", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); @@ -1162,6 +1162,13 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_tile_in ( topo, "rpc", 0UL, "metric_in", "votor_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); } + /* failov_votor has to come before the unpolled sign_votor, the votor looks + up in_kind by polled input index. */ + if( FD_UNLIKELY( failover_enabled ) ) { + /**/ fd_topob_tile_in ( topo, "votor", 0UL, "metric_in", "failov_votor", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "votor", 0UL, "votor_hist", 0UL ); + /**/ fd_topob_tile_out( topo, "votor", 0UL, "votor_failov", 0UL ); + } /**/ fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "votor_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "votor", 0UL, "votor_sign", 0UL ); /**/ fd_topob_tile_in ( topo, "votor", 0UL, "metric_in", "sign_votor", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED ); @@ -1730,6 +1737,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->votor.ip_addr = config->net.ip_addr; tile->votor.max_live_slots = config->firedancer.runtime.max_live_slots; fd_cstr_ncpy( tile->votor.identity_key_path, identity_key_path, sizeof(tile->votor.identity_key_path) ); + tile->votor.failover_standby = config->firedancer.failover.enabled; fd_cstr_ncpy( tile->votor.vote_history_path, config->paths.vote_history, sizeof(tile->votor.vote_history_path) ); tile->votor.target_gid = config->gid; tile->votor.target_uid = config->uid; diff --git a/src/app/shared/fd_config.c b/src/app/shared/fd_config.c index 9685723219d..f3c233ff449 100644 --- a/src/app/shared/fd_config.c +++ b/src/app/shared/fd_config.c @@ -540,11 +540,6 @@ fd_config_validate_failover( fd_config_t const * config ) { if( FD_UNLIKELY( !strcmp( config->paths.vote_account, "" ) ) ) { FD_LOG_ERR(( "[failover.enabled] is true, so [paths.vote_account] must be set to the vote account both machines vote for" )); } - /* Failover is built on the Tower tiles, and Votor does not adopt a - vote history or follow the staked key's BLS key through a switch. */ - if( FD_UNLIKELY( f->consensus.alpenglow ) ) { - FD_LOG_ERR(( "failover in this build supports Tower only, set [consensus.alpenglow] to false or [failover.enabled] to false" )); - } } void diff --git a/src/app/shared/test_config_parse.c b/src/app/shared/test_config_parse.c index 297f1e63c57..7b50e547cfc 100644 --- a/src/app/shared/test_config_parse.c +++ b/src/app/shared/test_config_parse.c @@ -242,7 +242,8 @@ main( int argc, FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 0, "vote.json", 0 ) ); FD_TEST( !failover_is_valid( config, 1, "localhost", 8010, "vote.json", 0 ) ); FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 8010, "", 0 ) ); - FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 8010, "vote.json", 1 ) ); + FD_TEST( failover_is_valid( config, 1, "0.0.0.0", 8010, "vote.json", 1 ) ); + FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 0, "vote.json", 1 ) ); FD_TEST( failover_is_valid( config, 0, "", 0, "", 1 ) ); /* Ensure we can selectively override a field */ diff --git a/src/choreo/votor/Local.mk b/src/choreo/votor/Local.mk index 8e5cfe59d98..8d705b07c88 100644 --- a/src/choreo/votor/Local.mk +++ b/src/choreo/votor/Local.mk @@ -1,5 +1,5 @@ -$(call add-hdrs,ag_votor_base.h ag_bls_serde.h ag_vote.h ag_vote_serde.h ag_epoch_info.h ag_cert.h ag_cert_serde.h ag_vote_history_serde.h ag_pool.h ag_slot_state.h ag_finality_tracker.h ag_parent_ready_tracker.h ag_votor.h) -$(call add-objs,ag_bls_serde ag_vote ag_vote_serde ag_epoch_info ag_cert ag_cert_serde ag_pool ag_slot_state ag_finality_tracker ag_parent_ready_tracker ag_votor ag_vote_history_serde,fd_choreo) +$(call add-hdrs,ag_votor_base.h ag_bls_serde.h ag_vote.h ag_vote_serde.h ag_epoch_info.h ag_cert.h ag_cert_serde.h ag_vote_history_serde.h ag_hist.h ag_pool.h ag_slot_state.h ag_finality_tracker.h ag_parent_ready_tracker.h ag_votor.h) +$(call add-objs,ag_bls_serde ag_vote ag_vote_serde ag_epoch_info ag_cert ag_cert_serde ag_pool ag_slot_state ag_finality_tracker ag_parent_ready_tracker ag_votor ag_vote_history_serde ag_hist,fd_choreo) ifdef FD_HAS_HOSTED $(call make-unit-test,test_ag_votor_base,test_ag_votor_base,fd_util) @@ -7,6 +7,7 @@ $(call make-unit-test,test_ag_bls,test_ag_bls,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_vote,test_ag_vote,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_epoch_info,test_ag_epoch_info,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_cert,test_ag_cert,fd_choreo fd_ballet fd_util) +$(call make-unit-test,test_ag_hist,test_ag_hist,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_pool,test_ag_pool,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_slot_state,test_ag_slot_state,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_finality_tracker,test_ag_finality_tracker,fd_choreo fd_ballet fd_util) @@ -14,15 +15,18 @@ $(call make-unit-test,test_ag_parent_ready_tracker,test_ag_parent_ready_tracker, $(call make-unit-test,test_ag_votor,test_ag_votor,fd_choreo fd_ballet fd_util) $(call make-unit-test,test_ag_vote_history_serde,test_ag_vote_history_serde,fd_choreo fd_ballet fd_util) $(call run-unit-test,test_ag_vote_history_serde) +$(call make-unit-test,test_ag_votor_failover,test_ag_votor_failover,fd_choreo fd_ballet fd_util) $(call run-unit-test,test_ag_votor_base) $(call run-unit-test,test_ag_bls) $(call run-unit-test,test_ag_vote) $(call run-unit-test,test_ag_epoch_info) $(call run-unit-test,test_ag_cert) +$(call run-unit-test,test_ag_hist) $(call run-unit-test,test_ag_pool) $(call run-unit-test,test_ag_slot_state) $(call run-unit-test,test_ag_finality_tracker) $(call run-unit-test,test_ag_parent_ready_tracker) $(call run-unit-test,test_ag_votor) +$(call run-unit-test,test_ag_votor_failover) $(call make-fuzz-test,fuzz_ag_vote_history_serde,fuzz_ag_vote_history_serde,fd_choreo fd_ballet fd_util) endif diff --git a/src/choreo/votor/ag_hist.c b/src/choreo/votor/ag_hist.c new file mode 100644 index 00000000000..fc160186713 --- /dev/null +++ b/src/choreo/votor/ag_hist.c @@ -0,0 +1,87 @@ +#include "ag_hist.h" + +/* rec_sz is the wire size of a record with these flags */ + +FD_FN_CONST static inline ulong +rec_sz( uchar flags ) { + return AG_HIST_REC_MIN_SZ + fd_ulong_if( flags & AG_HIST_FLAG_VOTED_NOTAR, sizeof(ag_block_hash_t), 0UL ); +} + +/* hist_ok checks the rules both ag_hist_ser and ag_hist_de enforce. */ + +static int +hist_ok( ag_hist_t const * hist ) { + if( FD_UNLIKELY( hist->rec_cnt>AG_HIST_MAX ) ) return 0; + if( FD_UNLIKELY( hist->anchor==ULONG_MAX ) ) return 0; + ulong first = ag_hist_first_slot( hist->anchor ); + ulong prev = ULONG_MAX; + for( ulong i=0UL; irec_cnt; i++ ) { + ag_hist_rec_t const * rec = &hist->rec[ i ]; + if( FD_UNLIKELY( rec->slotslot<=prev ) ) return 0; + if( FD_UNLIKELY( rec->flags & ~AG_HIST_FLAG_MASK ) ) return 0; + if( FD_UNLIKELY( !( rec->flags & AG_HIST_FLAG_VOTED ) ) ) return 0; + prev = rec->slot; + } + return 1; +} + +int +ag_hist_ser( ag_hist_t const * hist, + uchar * buf, + ulong buf_max, + ulong * out_sz ) { + if( FD_UNLIKELY( !hist_ok( hist ) ) ) return -1; + + ulong sz = AG_HIST_HDR_SZ; + for( ulong i=0UL; irec_cnt; i++ ) sz += rec_sz( hist->rec[ i ].flags ); + if( FD_UNLIKELY( sz>buf_max ) ) return -1; + + ulong off = 0UL; + FD_STORE( ulong, buf+off, hist->anchor ); off += sizeof(ulong); + FD_STORE( ulong, buf+off, hist->last_leader_slot ); off += sizeof(ulong); + FD_STORE( ulong, buf+off, hist->vote_bound ); off += sizeof(ulong); + FD_STORE( ushort, buf+off, (ushort)hist->rec_cnt ); off += sizeof(ushort); + for( ulong i=0UL; irec_cnt; i++ ) { + ag_hist_rec_t const * rec = &hist->rec[ i ]; + FD_STORE( ulong, buf+off, rec->slot ); off += sizeof(ulong); + buf[ off ] = rec->flags; off += sizeof(uchar); + if( rec->flags & AG_HIST_FLAG_VOTED_NOTAR ) { fd_memcpy( buf+off, rec->notar_hash, sizeof(ag_block_hash_t) ); off += sizeof(ag_block_hash_t); } + } + + *out_sz = off; + return 0; +} + +int +ag_hist_de( uchar const * buf, + ulong buf_sz, + ag_hist_t * out ) { + if( FD_UNLIKELY( buf_szanchor = FD_LOAD( ulong, buf+off ); off += sizeof(ulong); + hist->last_leader_slot = FD_LOAD( ulong, buf+off ); off += sizeof(ulong); + hist->vote_bound = FD_LOAD( ulong, buf+off ); off += sizeof(ulong); + hist->rec_cnt = FD_LOAD( ushort, buf+off ); off += sizeof(ushort); + if( FD_UNLIKELY( hist->rec_cnt>AG_HIST_MAX ) ) return -1; + + for( ulong i=0UL; irec_cnt; i++ ) { + ag_hist_rec_t * rec = &hist->rec[ i ]; + if( FD_UNLIKELY( buf_sz-offslot = FD_LOAD( ulong, buf+off ); off += sizeof(ulong); + rec->flags = buf[ off ]; off += sizeof(uchar); + if( rec->flags & AG_HIST_FLAG_VOTED_NOTAR ) { + if( FD_UNLIKELY( buf_sz-offnotar_hash, buf+off, sizeof(ag_block_hash_t) ); off += sizeof(ag_block_hash_t); + } + } + if( FD_UNLIKELY( off!=buf_sz ) ) return -1; /* trailing bytes */ + if( FD_UNLIKELY( !hist_ok( hist ) ) ) return -1; + + *out = *hist; + return 0; +} diff --git a/src/choreo/votor/ag_hist.h b/src/choreo/votor/ag_hist.h new file mode 100644 index 00000000000..da9745b9d89 --- /dev/null +++ b/src/choreo/votor/ag_hist.h @@ -0,0 +1,75 @@ +#ifndef HEADER_fd_src_choreo_votor_ag_hist_h +#define HEADER_fd_src_choreo_votor_ag_hist_h + +/* ag_hist is a validator's own Alpenglow votes since a finality anchor, + which a failover peer adopts before it votes. */ + +#include "ag_votor_base.h" + +#define AG_HIST_MAX (128UL) /* records per history, 32 leader windows */ + +#define AG_HIST_FLAG_VOTED (1U) /* any own vote on the slot */ +#define AG_HIST_FLAG_VOTED_NOTAR (2U) /* a notar vote, notar_hash is set */ +#define AG_HIST_FLAG_BAD_WINDOW (4U) /* no final vote may follow on this slot */ +#define AG_HIST_FLAG_RETIRED (8U) /* the slot is done */ +#define AG_HIST_FLAG_MASK (15U) + +#define AG_HIST_HDR_SZ (26UL) /* anchor u64, last_leader_slot u64, vote_bound u64, rec_cnt u16 */ +#define AG_HIST_REC_MIN_SZ (9UL) /* slot u64, flags u8 */ +#define AG_HIST_REC_MAX_SZ (41UL) /* plus the 32 byte notar hash */ +#define AG_HIST_SER_MAX (AG_HIST_HDR_SZ+AG_HIST_MAX*AG_HIST_REC_MAX_SZ) /* 5274 */ + +struct ag_hist_rec { + ulong slot; + uchar flags; /* AG_HIST_FLAG_* */ + ag_block_hash_t notar_hash; /* only meaningful when AG_HIST_FLAG_VOTED_NOTAR is set */ +}; +typedef struct ag_hist_rec ag_hist_rec_t; + +struct ag_hist { + ulong anchor; /* exporter's highest_final_cert_slot, raised when the export was truncated */ + ulong last_leader_slot; /* slot of the last LEADER the exporter published, ULONG_MAX when none */ + ulong vote_bound; /* the exporter casts no vote at or below this slot, ULONG_MAX when none */ + ulong rec_cnt; + ag_hist_rec_t rec[ AG_HIST_MAX ]; /* ascending slots */ +}; +typedef struct ag_hist ag_hist_t; + +FD_PROTOTYPES_BEGIN + +/* ag_hist_first_slot returns the lowest slot a history with this anchor + may contain. */ + +FD_FN_CONST static inline ulong +ag_hist_first_slot( ulong anchor ) { + return ag_first_slot_in_window( fd_ulong_sat_sub( anchor, AG_REWARD_SLOT_DELTA ) ); +} + +/* ag_hist_ser serializes hist into buf. Returns 0 on success, -1 if hist + is invalid or buf_max is too small. */ + +int +ag_hist_ser( ag_hist_t const * hist, + uchar * buf, + ulong buf_max, + ulong * out_sz ); + +/* ag_hist_de deserializes exactly buf_sz bytes of buf into out. Returns 0 + on success, -1 if the bytes are not a valid history. */ + +int +ag_hist_de( uchar const * buf, + ulong buf_sz, + ag_hist_t * out ); + +/* ag_hist_tip returns the highest record slot, ULONG_MAX when the + history is empty. */ + +FD_FN_PURE static inline ulong +ag_hist_tip( ag_hist_t const * hist ) { + return hist->rec_cnt ? hist->rec[ hist->rec_cnt-1UL ].slot : ULONG_MAX; +} + +FD_PROTOTYPES_END + +#endif /* HEADER_fd_src_choreo_votor_ag_hist_h */ diff --git a/src/choreo/votor/ag_votor.c b/src/choreo/votor/ag_votor.c index 1045bbaad09..d2e5c0c1d6c 100644 --- a/src/choreo/votor/ag_votor.c +++ b/src/choreo/votor/ag_votor.c @@ -24,6 +24,8 @@ struct slot_state_ele { ag_block_info_t pending_block_info; int retired; + uchar adopted_notar; /* voted_notar came from an adopted history, failover only */ + long timeout; struct { ulong prev; ulong next; } pending_dlist; @@ -90,6 +92,7 @@ struct __attribute__((aligned(128UL))) ag_votor { fd_bls_sign_fn bls_sign_fn; void * bls_sign_ctx; + ulong vote_bound; /* we cast no vote at or below this slot, ULONG_MAX when none */ epoch_bls_key_t prev_epoch; epoch_bls_key_t curr_epoch; @@ -276,6 +279,7 @@ ag_votor_new( void * mem, votor->prev_epoch = (epoch_bls_key_t){ .start_slot = ULONG_MAX, .rank = USHORT_MAX }; votor->curr_epoch = (epoch_bls_key_t){ .start_slot = ULONG_MAX, .rank = USHORT_MAX }; votor->next_epoch = (epoch_bls_key_t){ .start_slot = ULONG_MAX, .rank = USHORT_MAX }; + votor->vote_bound = ULONG_MAX; votor->vote_events = vote_events_join( vote_events_new( vote_events, events_max ) ); votor->pending_dlist = pending_dlist_join( pending_dlist_new( pending_dlist ) ); for( ulong k=0UL; ktimeout_dlist[ k ] = timeout_dlist_join( timeout_dlist_new( timeout_dlist+k*timeout_dlist_footprint() ) ); @@ -403,6 +407,14 @@ first_unpruned_slot( ag_votor_t const * self ) { return ag_first_slot_in_window( fd_ulong_sat_sub( self->highest_final_cert_slot, AG_REWARD_SLOT_DELTA ) ); } +/* Returns whether slot is at or below our vote bound. */ + +FD_FN_PURE static int +below_vote_bound( ag_votor_t const * self, + ulong slot ) { + return self->vote_bound!=ULONG_MAX && slot<=self->vote_bound; +} + static int should_ignore_pool_event( ag_votor_t const * self, ag_pool_event_t const * event ) { @@ -428,6 +440,7 @@ try_final( ag_votor_t * self, ulong slot, ag_block_hash_t const hash ) { FD_TEST( slot>=first_unpruned_slot( self ) ); + if( FD_UNLIKELY( below_vote_bound( self, slot ) ) ) return; epoch_bls_key_t const * epoch = own_epoch( self, slot ); slot_state_ele_t const * state = slot_state_map_ele_query_const( self->slot_states->map, &slot, NULL, self->slot_states->pool ); @@ -450,6 +463,7 @@ try_notar( ag_votor_t * self, FD_TEST( slot>=first_unpruned_slot( self ) ); epoch_bls_key_t const * epoch = own_epoch( self, slot ); if( FD_UNLIKELY( has_voted( self, slot ) ) ) return 0; + if( FD_UNLIKELY( below_vote_bound( self, slot ) ) ) return 0; ag_block_hash_t hash; memcpy( hash, block_info->hash, sizeof(ag_block_hash_t) ); @@ -489,6 +503,7 @@ try_skip_window( ag_votor_t * self, ulong window_start = ag_first_slot_in_window( slot ); for( ulong s=window_start; svoted = 1; @@ -605,6 +620,144 @@ epoch_starting_at( ag_votor_t * self, FD_LOG_CRIT(( "no epoch starts at slot %lu", epoch_slot )); } +void +ag_votor_set_vote_bound( ag_votor_t * self, + ulong bound ) { + self->vote_bound = self->vote_bound==ULONG_MAX ? bound : fd_ulong_max( self->vote_bound, bound ); +} + +ulong +ag_votor_vote_bound( ag_votor_t const * self ) { + return self->vote_bound; +} + +void +ag_votor_mark_unsent( ag_votor_t * self, + ag_vote_t const * vote ) { + ulong slot = ag_vote_slot( vote ); + if( FD_UNLIKELY( self->highest_final_cert_slot==ULONG_MAX || slotkind==AG_VOTE_KIND_NOTAR && state->adopted_notar ) ) return; + state->voted = 1; + state->bad_window = 1; + /* Clear the notar mark of a dropped notar so the slot cannot become a + parent. */ + if( FD_UNLIKELY( vote->kind==AG_VOTE_KIND_NOTAR ) ) { + state->voted_notar = 0; + fd_memset( state->voted_notar_hash, 0, sizeof(ag_block_hash_t) ); + } +} + +/* advance_root raises the finality anchor to slot and prunes below it, if + slot is past the anchor. */ + +static void +advance_root( ag_votor_t * self, + ulong slot ) { + if( FD_UNLIKELY( self->highest_final_cert_slot==ULONG_MAX || slot<=self->highest_final_cert_slot ) ) return; + self->highest_final_cert_slot = slot; + prune( self ); +} + +int +ag_votor_hist_export( ag_votor_t * self, + ulong last_leader_slot, + ag_hist_t * out ) { + out->last_leader_slot = last_leader_slot; + out->vote_bound = self->vote_bound; + out->rec_cnt = 0UL; + if( FD_UNLIKELY( self->highest_final_cert_slot==ULONG_MAX ) ) { + out->anchor = 0UL; + return 0; + } + + slot_state_map_t * map = self->slot_states->map; + slot_state_ele_t * pool = self->slot_states->pool; + ulong * slots = self->scratch.slots; + ulong first = first_unpruned_slot( self ); + ulong cnt = 0UL; + for( slot_state_map_iter_t iter = slot_state_map_iter_init( map, pool ); + !slot_state_map_iter_done( iter, map, pool ); + iter = slot_state_map_iter_next( iter, map, pool ) ) { + slot_state_ele_t const * ele = slot_state_map_iter_ele_const( iter, map, pool ); + if( FD_LIKELY( ele->voted && ele->slot>=first && cntslot_max ) ) slots[ cnt++ ] = ele->slot; + } + slot_sort_inplace( slots, cnt ); + + /* If there are too many voted slots, we drop whole windows from the + bottom and raise the anchor past them. */ + ulong lo = 0UL; + while( cnt-lo>AG_HIST_MAX ) { + ulong next_window = ag_first_slot_in_window( slots[ lo ] )+AG_SLOTS_PER_WINDOW; + while( lo0UL; + out->anchor = self->highest_final_cert_slot; + if( FD_UNLIKELY( truncated ) ) out->anchor = fd_ulong_max( out->anchor, ag_first_slot_in_window( slots[ lo ] )+AG_REWARD_SLOT_DELTA ); + + for( ulong i=lo; irec[ out->rec_cnt++ ]; + rec->slot = ele->slot; + rec->flags = (uchar)( AG_HIST_FLAG_VOTED | + fd_uint_if( ele->voted_notar, AG_HIST_FLAG_VOTED_NOTAR, 0U ) | + fd_uint_if( ele->bad_window, AG_HIST_FLAG_BAD_WINDOW, 0U ) | + fd_uint_if( ele->retired, AG_HIST_FLAG_RETIRED, 0U ) ); + if( ele->voted_notar ) fd_memcpy( rec->notar_hash, ele->voted_notar_hash, sizeof(ag_block_hash_t) ); + else fd_memset( rec->notar_hash, 0, sizeof(ag_block_hash_t) ); + } + return truncated; +} + +ulong +ag_votor_hist_adopt( ag_votor_t * self, + ag_hist_t const * hist ) { + slot_state_map_t * map = self->slot_states->map; + slot_state_ele_t * pool = self->slot_states->pool; + ulong first = ag_hist_first_slot( fd_ulong_max( self->highest_final_cert_slot, hist->anchor ) ); + ulong free = slot_state_pool_free( pool ); + if( hist->anchor>self->highest_final_cert_slot ) { + for( slot_state_map_iter_t iter = slot_state_map_iter_init( map, pool ); + !slot_state_map_iter_done( iter, map, pool ); + iter = slot_state_map_iter_next( iter, map, pool ) ) { + ulong slot = slot_state_map_iter_ele_const( iter, map, pool )->slot; + free += slot>=self->root && slotrec_cnt; i++ ) { + ulong slot = hist->rec[ i ].slot; + if( slotanchor ); + /* Take the exporter's vote bound as well. */ + if( FD_UNLIKELY( hist->vote_bound!=ULONG_MAX ) ) ag_votor_set_vote_bound( self, hist->vote_bound ); + ulong conflicts = 0UL; + for( ulong i=0UL; irec_cnt; i++ ) { + ag_hist_rec_t const * rec = &hist->rec[ i ]; + if( FD_UNLIKELY( rec->slotslot ); + if( rec->flags & AG_HIST_FLAG_VOTED_NOTAR ) { + if( FD_UNLIKELY( state->voted_notar && !fd_memeq( state->voted_notar_hash, rec->notar_hash, sizeof(ag_block_hash_t) ) ) ) conflicts++; + state->voted_notar = 1; + state->adopted_notar = 1; + fd_memcpy( state->voted_notar_hash, rec->notar_hash, sizeof(ag_block_hash_t) ); + } + if( !state->voted ) { + if( FD_UNLIKELY( state->pending_block ) ) pending_dlist_ele_remove( self->pending_dlist, state, self->slot_states->pool ); + state->pending_block = 0; + state->voted = 1; + } + state->bad_window |= !!( rec->flags & AG_HIST_FLAG_BAD_WINDOW ); + state->retired |= !!( rec->flags & AG_HIST_FLAG_RETIRED ); + } + return conflicts; +} + void ag_votor_set_bls_key( ag_votor_t * self, ulong epoch_slot, @@ -664,6 +817,8 @@ ag_votor_handle_pool_event( ag_votor_t * self, case AG_POOL_EVENT_SAFE_TO_NOTAR: { ulong slot = event->safe_to_notar.slot; uchar const * hash = event->safe_to_notar.hash; + /* No fallback vote at or below the vote bound. */ + if( FD_UNLIKELY( below_vote_bound( self, slot ) ) ) { try_skip_window( self, slot, AG_VOTOR_REASON_SAFE_TO_NOTAR ); break; } epoch_bls_key_t const * epoch = own_epoch( self, slot ); if( FD_LIKELY( epoch->has_bls_key && slot>=self->catchup_vote_slot ) ) { ag_vote_t vote = ag_vote_construct_notar_fallback( self->bls_sign_fn, self->bls_sign_ctx, epoch->bls_key, slot, hash, (ushort)epoch->rank, self->shred_version ); @@ -676,6 +831,8 @@ ag_votor_handle_pool_event( ag_votor_t * self, case AG_POOL_EVENT_SAFE_TO_SKIP: { ulong slot = event->safe_to_skip; + /* No fallback vote at or below the vote bound. */ + if( FD_UNLIKELY( below_vote_bound( self, slot ) ) ) { try_skip_window( self, slot, AG_VOTOR_REASON_SAFE_TO_SKIP ); break; } epoch_bls_key_t const * epoch = own_epoch( self, slot ); if( FD_LIKELY( epoch->has_bls_key && slot>=self->catchup_vote_slot ) ) { ag_vote_t vote = ag_vote_construct_skip_fallback( self->bls_sign_fn, self->bls_sign_ctx, epoch->bls_key, slot, (ushort)epoch->rank, self->shred_version ); diff --git a/src/choreo/votor/ag_votor.h b/src/choreo/votor/ag_votor.h index ebe13ca83ce..8ac103f4bb5 100644 --- a/src/choreo/votor/ag_votor.h +++ b/src/choreo/votor/ag_votor.h @@ -4,6 +4,7 @@ #include "ag_votor_base.h" #include "../../ballet/bls/fd_bls.h" #include "ag_pool.h" +#include "ag_hist.h" #define AG_VOTOR_REASON_BLOCK_REPLAYED (0) #define AG_VOTOR_REASON_PARENT_READY (1) @@ -89,6 +90,40 @@ ag_votor_set_rank( ag_votor_t * self, ulong epoch_slot, ulong epoch_rank ); +/* ag_votor_set_vote_bound stops us voting at or below bound, which only + moves up. ag_votor_vote_bound returns it, ULONG_MAX when none is set. */ + +void +ag_votor_set_vote_bound( ag_votor_t * self, + ulong bound ); + +FD_FN_PURE ulong +ag_votor_vote_bound( ag_votor_t const * self ); + +/* ag_votor_mark_unsent records that a vote we built never left the + machine, so no final vote follows on its slot. */ + +void +ag_votor_mark_unsent( ag_votor_t * self, + ag_vote_t const * vote ); + +/* ag_votor_hist_export writes our own votes since the anchor into out. + Returns 1 if it had to drop old windows to fit. */ + +int +ag_votor_hist_export( ag_votor_t * self, + ulong last_leader_slot, + ag_hist_t * out ); + +/* ag_votor_hist_adopt merges a valid history into initialized slot states. + Returns how many notar hashes disagreed with ours, or ULONG_MAX if the + retained local and imported slots exceed capacity. Failure leaves + consensus state unchanged. */ + +ulong +ag_votor_hist_adopt( ag_votor_t * self, + ag_hist_t const * hist ); + /* Algorithm 1, lines 9-25. Votor::handle_pool_event */ ag_cert_t const * diff --git a/src/choreo/votor/test_ag_hist.c b/src/choreo/votor/test_ag_hist.c new file mode 100644 index 00000000000..4428dcfc1d7 --- /dev/null +++ b/src/choreo/votor/test_ag_hist.c @@ -0,0 +1,25 @@ +#include "ag_hist.h" + +int +main( int argc, char ** argv ) { + fd_boot( &argc, &argv ); + ag_hist_t hist = { .anchor=100UL, .last_leader_slot=ULONG_MAX, .vote_bound=99UL, .rec_cnt=1UL }; + hist.rec[0].slot = 101UL; + hist.rec[0].flags = AG_HIST_FLAG_VOTED|AG_HIST_FLAG_VOTED_NOTAR; + fd_memset( hist.rec[0].notar_hash, 0x42, sizeof(ag_block_hash_t) ); + uchar buf[AG_HIST_SER_MAX]; + ulong sz; + FD_TEST( !ag_hist_ser( &hist, buf, sizeof(buf), &sz ) ); + FD_TEST( sz==AG_HIST_HDR_SZ+AG_HIST_REC_MAX_SZ ); + ag_hist_t out; + FD_TEST( !ag_hist_de( buf, sz, &out ) ); + FD_TEST( out.anchor==hist.anchor && out.rec_cnt==1UL && out.last_leader_slot==ULONG_MAX && out.vote_bound==99UL ); + FD_TEST( out.rec[0].slot==101UL && out.rec[0].flags==hist.rec[0].flags ); + FD_TEST( fd_memeq( out.rec[0].notar_hash, hist.rec[0].notar_hash, sizeof(ag_block_hash_t) ) ); + ag_hist_t saved = out; + FD_TEST( ag_hist_de( buf, sz-1UL, &out )==-1 ); + FD_TEST( fd_memeq( &out, &saved, sizeof(out) ) ); + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} diff --git a/src/choreo/votor/test_ag_votor_failover.c b/src/choreo/votor/test_ag_votor_failover.c new file mode 100644 index 00000000000..cf5033f2481 --- /dev/null +++ b/src/choreo/votor/test_ag_votor_failover.c @@ -0,0 +1,293 @@ +#include "ag_votor.c" +#include "test_ag_cert_builder.h" + +#define NV (2UL) +#define TEST_SLOT_MAX (16UL) +#define TEST_SHRED_VERSION ((ushort)0x5a5a) + +#define TEST_NS_PER_SLOT (400000000L) + +#define FD_TEST_NO_MSG( votor ) do { \ + ag_vote_t unused_; \ + FD_TEST( !try_recv( (votor), &unused_ ) ); \ + } while( 0 ) + +#define SCRATCH_MAX (1UL<<23) + +static uchar scratch_a[ SCRATCH_MAX ] __attribute__((aligned(128))); +static uchar scratch_b[ SCRATCH_MAX ] __attribute__((aligned(128))); + +static fd_bls_sec_t g_sk[ NV ]; +static ulong g_hash_ctr = 0UL; + +static void +genesis_hash( ag_block_hash_t out ) { + fd_memset( out, 0, sizeof(ag_block_hash_t) ); +} + +static void +random_hash( ag_block_hash_t out ) { + fd_memset( out, 0, sizeof(ag_block_hash_t) ); + FD_STORE( ulong, out, 0x9000UL + (++g_hash_ctr) ); + FD_STORE( ulong, out+8UL, 0xc0ffee00UL ^ g_hash_ctr ); +} + +static ag_block_id_t +genesis_block_id( void ) { + ag_block_id_t b; b.slot = 0UL; genesis_hash( b.hash ); + return b; +} + +static void +create_validators( void ) { + for( ulong i=0UL; irec_cnt ); + ag_hist_rec_t const * rec = &hist->rec[ idx ]; + FD_TEST( rec->slot ==slot ); + FD_TEST( rec->flags==(uchar)flags ); + if( notar_hash ) FD_TEST( fd_memeq( rec->notar_hash, notar_hash, sizeof(ag_block_hash_t) ) ); +} + +static void +round_trip( ag_hist_t const * hist ) { + uchar buf[ AG_HIST_SER_MAX ]; + ulong sz = 0UL; + FD_TEST( !ag_hist_ser( hist, buf, sizeof(buf), &sz ) ); + + ag_hist_t out; fd_memset( &out, 0xAA, sizeof(ag_hist_t) ); + FD_TEST( !ag_hist_de( buf, sz, &out ) ); + FD_TEST( out.anchor ==hist->anchor ); + FD_TEST( out.last_leader_slot==hist->last_leader_slot ); + FD_TEST( out.vote_bound ==hist->vote_bound ); + FD_TEST( out.rec_cnt ==hist->rec_cnt ); + for( ulong i=0UL; irec_cnt; i++ ) { + FD_TEST( out.rec[ i ].slot ==hist->rec[ i ].slot ); + FD_TEST( out.rec[ i ].flags==hist->rec[ i ].flags ); + if( hist->rec[ i ].flags & AG_HIST_FLAG_VOTED_NOTAR ) FD_TEST( fd_memeq( out.rec[ i ].notar_hash, hist->rec[ i ].notar_hash, sizeof(ag_block_hash_t) ) ); + } +} + +static void +test_export_shape( void ) { + ag_votor_t * votor = setup_votor( scratch_a, 0L ); + ag_block_id_t parent = genesis_block_id(); + + ag_block_hash_t hash[ 4 ]; + genesis_hash( hash[ 0 ] ); + for( ulong s=1UL; s<=3UL; s++ ) { + ag_vote_t vote = send_block_and_expect_notar( votor, s, &parent ); + fd_memcpy( hash[ s ], vote.notar.block_hash, sizeof(ag_block_hash_t) ); + parent = ag_block_id( s, hash[ s ] ); + } + FD_TEST_NO_MSG( votor ); + + ag_hist_t hist; + FD_TEST( !ag_votor_hist_export( votor, 42UL, &hist ) ); + FD_TEST( hist.anchor ==0UL ); + FD_TEST( hist.last_leader_slot==42UL ); + FD_TEST( hist.vote_bound ==ULONG_MAX ); + FD_TEST( hist.rec_cnt ==4UL ); + expect_rec( &hist, 0UL, 0UL, AG_HIST_FLAG_VOTED|AG_HIST_FLAG_VOTED_NOTAR|AG_HIST_FLAG_RETIRED, hash[ 0 ] ); + for( ulong s=1UL; s<=3UL; s++ ) expect_rec( &hist, s, s, AG_HIST_FLAG_VOTED|AG_HIST_FLAG_VOTED_NOTAR, hash[ s ] ); + round_trip( &hist ); + + teardown_votor( votor ); + FD_LOG_NOTICE(( "pass: export_shape" )); +} + +static void +test_mark_unsent_notar( void ) { + ag_votor_t * votor = setup_votor( scratch_a, 0L ); + ag_block_id_t parent = genesis_block_id(); + + ag_vote_t notar = send_block_and_expect_notar( votor, 1UL, &parent ); + ag_block_hash_t h1; fd_memcpy( h1, notar.notar.block_hash, sizeof(ag_block_hash_t) ); + FD_TEST_NO_MSG( votor ); + + /* A discarded notar must not appear as sent or become a parent. */ + ag_votor_mark_unsent( votor, ¬ar ); + + ag_block_hash_t zero; fd_memset( zero, 0, sizeof(ag_block_hash_t) ); + ag_hist_t hist; + FD_TEST( !ag_votor_hist_export( votor, 42UL, &hist ) ); + expect_rec( &hist, 1UL, 1UL, AG_HIST_FLAG_VOTED|AG_HIST_FLAG_BAD_WINDOW, zero ); + + ag_block_info_t block = {0}; + random_hash( block.hash ); + block.parent = ag_block_id( 1UL, h1 ); + ag_votor_process_replay( votor, 2UL, &block ); + FD_TEST_NO_MSG( votor ); + + teardown_votor( votor ); + + FD_LOG_NOTICE(( "pass: mark_unsent_notar" )); +} + +/* Test that the exporter's vote bound goes out with its history and holds + on the adopter. */ + +static void +test_bound_travels( void ) { + ag_votor_t * a = setup_votor( scratch_a, 0L ); + ag_block_id_t parent = genesis_block_id(); + ag_vote_t notar = send_block_and_expect_notar( a, 1UL, &parent ); + ag_block_id_t tip = ag_block_id( 1UL, notar.notar.block_hash ); + FD_TEST_NO_MSG( a ); + + ag_hist_t plain; + FD_TEST( !ag_votor_hist_export( a, ULONG_MAX, &plain ) ); + FD_TEST( plain.vote_bound==ULONG_MAX ); + ag_votor_set_vote_bound( a, 5UL ); + ag_hist_t bounded; + FD_TEST( !ag_votor_hist_export( a, ULONG_MAX, &bounded ) ); + FD_TEST( bounded.vote_bound==5UL ); + round_trip( &bounded ); + teardown_votor( a ); + + /* Without a vote bound the adopted notar on slot 1 lets slot 2 vote. */ + ag_votor_t * b = setup_votor( scratch_b, 0L ); + FD_TEST( !ag_votor_hist_adopt( b, &plain ) ); + FD_TEST( ag_votor_vote_bound( b )==ULONG_MAX ); + send_block_and_expect_notar( b, 2UL, &tip ); + teardown_votor( b ); + + b = setup_votor( scratch_b, 0L ); + FD_TEST( !ag_votor_hist_adopt( b, &bounded ) ); + FD_TEST( ag_votor_vote_bound( b )==5UL ); + ag_block_info_t block = {0}; + random_hash( block.hash ); + block.parent = tip; + ag_votor_process_replay( b, 2UL, &block ); + FD_TEST_NO_MSG( b ); + + /* A lower vote bound from a later history does not lower ours. */ + bounded.vote_bound = 3UL; + ag_votor_hist_adopt( b, &bounded ); + FD_TEST( ag_votor_vote_bound( b )==5UL ); + teardown_votor( b ); + + FD_LOG_NOTICE(( "pass: bound_travels" )); +} + +static void +expect_adopt_full( ag_votor_t * votor, + ag_hist_t const * hist ) { + round_trip( hist ); + ulong footprint = ag_votor_footprint( TEST_SLOT_MAX ); + fd_memcpy( scratch_b, scratch_a, footprint ); + FD_TEST( ag_votor_hist_adopt( votor, hist )==ULONG_MAX ); + FD_TEST( fd_memeq( scratch_a, scratch_b, footprint ) ); +} + +static void +test_adopt_capacity( void ) { + ag_votor_t * votor = setup_votor( scratch_a, 0L ); + for( ulong slot=1UL; slotslot_states->pool ) ); + + ag_hist_t hist = { .anchor = 12UL, .last_leader_slot = 80UL, .vote_bound = 83UL, .rec_cnt = AG_HIST_MAX }; + for( ulong i=0UL; iroot==4UL && votor->highest_final_cert_slot==12UL ); + FD_TEST( ag_votor_vote_bound( votor )==83UL ); + FD_TEST( !slot_state_pool_free( votor->slot_states->pool ) ); + FD_TEST( !ag_votor_hist_adopt( votor, &hist ) ); + + hist.rec_cnt++; + expect_adopt_full( votor, &hist ); + advance_root( votor, 16UL ); + FD_TEST( !ag_votor_hist_adopt( votor, &hist ) ); + FD_TEST( votor->root==8UL && votor->highest_final_cert_slot==16UL ); + FD_TEST( slot_state_pool_free( votor->slot_states->pool )==3UL ); + FD_TEST( !slot_state_map_ele_query_const( votor->slot_states->map, &hist.rec[ 0 ].slot, NULL, votor->slot_states->pool ) ); + FD_TEST( has_voted( votor, 20UL ) ); + + teardown_votor( votor ); + FD_LOG_NOTICE(( "pass: adopt_capacity" )); +} + +int +main( int argc, + char ** argv ) { + fd_boot( &argc, &argv ); + + test_export_shape(); + test_mark_unsent_notar(); + test_bound_travels(); + test_adopt_capacity(); + FD_LOG_NOTICE(( "pass" )); + fd_halt(); + return 0; +} diff --git a/src/disco/keyguard/fd_keyswitch.h b/src/disco/keyguard/fd_keyswitch.h index b546a1cb3c5..81f54715aca 100644 --- a/src/disco/keyguard/fd_keyswitch.h +++ b/src/disco/keyguard/fd_keyswitch.h @@ -39,7 +39,7 @@ struct __attribute__((aligned(FD_KEYSWITCH_ALIGN))) fd_keyswitch_private { ulong result; ulong param; uchar bytes[ 32768UL ]; - ulong operator; /* 1 when set-identity made this switch under failover, so the tower or votor votes without waiting for an adoption */ + ulong operator; /* 1 when set-identity made this switch under failover, so votor can participate without failover adoption */ /* Padding to FD_KEYSWITCH_ALIGN here */ }; diff --git a/src/disco/metrics/generate/types.py b/src/disco/metrics/generate/types.py index 07e03bbf9e5..43baa73b277 100644 --- a/src/disco/metrics/generate/types.py +++ b/src/disco/metrics/generate/types.py @@ -47,7 +47,6 @@ class Tile(Enum): MLX5 = 41 WAKER = 42 MWAITX = 43 - FAILOV = 44 SNAPMK = 50 SNAPZP = 51 diff --git a/src/disco/metrics/generated/fd_metrics_all.c b/src/disco/metrics/generated/fd_metrics_all.c index 8388a5aff15..d494ab83a18 100644 --- a/src/disco/metrics/generated/fd_metrics_all.c +++ b/src/disco/metrics/generated/fd_metrics_all.c @@ -1607,24 +1607,6 @@ const fd_metrics_meta_t FD_METRICS_MWAITX[FD_METRICS_MWAITX_TOTAL] = { DECLARE_METRIC_ENUM( MWAITX_WAKE_ISSUED, COUNTER, MWAITX_WAKE_CAUSE, SWEEP ), }; -const fd_metrics_meta_t FD_METRICS_FAILOV[FD_METRICS_FAILOV_TOTAL] = { - DECLARE_METRIC( FAILOV_CONN_ACCEPTED, COUNTER ), - DECLARE_METRIC( FAILOV_ACCEPT_BUDGET_EXHAUSTED, COUNTER ), - DECLARE_METRIC_ENUM( FAILOV_CONN_REJECTED, COUNTER, FAILOVER_CONN_REJECT_REASON, SOURCE ), - DECLARE_METRIC_ENUM( FAILOV_CONN_REJECTED, COUNTER, FAILOVER_CONN_REJECT_REASON, BUDGET ), - DECLARE_METRIC_ENUM( FAILOV_CONN_REJECTED, COUNTER, FAILOVER_CONN_REJECT_REASON, CAPACITY ), - DECLARE_METRIC( FAILOV_CONN_RECONNECT_ADMITTED, COUNTER ), - DECLARE_METRIC( FAILOV_HANDSHAKE_TIMED_OUT, COUNTER ), - DECLARE_METRIC( FAILOV_PROOF_REJECTED, COUNTER ), - DECLARE_METRIC( FAILOV_HELLO_REJECTED, COUNTER ), - DECLARE_METRIC( FAILOV_TLS_HANDSHAKE_FAILED, COUNTER ), - DECLARE_METRIC( FAILOV_FRAME_PROTOCOL_REJECTED, COUNTER ), - DECLARE_METRIC( FAILOV_CONN_AUTHENTICATED, COUNTER ), - DECLARE_METRIC( FAILOV_HANDSHAKE_DURATION_NANOS, COUNTER ), - DECLARE_METRIC( FAILOV_HANDSHAKE_LATEST_DURATION_NANOS, GAUGE ), - DECLARE_METRIC( FAILOV_CONN_PENDING, GAUGE ), -}; - const fd_metrics_meta_t FD_METRICS_SNAPMK[FD_METRICS_SNAPMK_TOTAL] = { DECLARE_METRIC( SNAPMK_JOB_STATE, GAUGE ), DECLARE_METRIC_ENUM( SNAPMK_SNAPSHOT_STARTED, COUNTER, SNAP_TYPE, FULL ), @@ -1986,7 +1968,6 @@ const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT] = { "mlx5", "waker", "mwaitx", - "failov", "snapmk", "snapzp", "snaprd", @@ -2033,7 +2014,6 @@ const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT] = { FD_METRICS_MLX5_TOTAL, FD_METRICS_WAKER_TOTAL, FD_METRICS_MWAITX_TOTAL, - FD_METRICS_FAILOV_TOTAL, FD_METRICS_SNAPMK_TOTAL, FD_METRICS_SNAPZP_TOTAL, FD_METRICS_SNAPRD_TOTAL, @@ -2079,7 +2059,6 @@ const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT] FD_METRICS_MLX5, FD_METRICS_WAKER, FD_METRICS_MWAITX, - FD_METRICS_FAILOV, FD_METRICS_SNAPMK, FD_METRICS_SNAPZP, FD_METRICS_SNAPRD, diff --git a/src/disco/metrics/generated/fd_metrics_all.h b/src/disco/metrics/generated/fd_metrics_all.h index be89f95d595..7481258cadf 100644 --- a/src/disco/metrics/generated/fd_metrics_all.h +++ b/src/disco/metrics/generated/fd_metrics_all.h @@ -90,7 +90,6 @@ enum { #include "fd_metrics_admin.h" #include "fd_metrics_waker.h" #include "fd_metrics_mwaitx.h" -#include "fd_metrics_failov.h" #include "fd_metrics_ipecho.h" #include "fd_metrics_backt.h" #include "fd_metrics_execrp.h" @@ -251,7 +250,7 @@ extern const fd_metrics_meta_t FD_METRICS_ALL_LINK_IN[FD_METRICS_ALL_LINK_IN_TOT #define FD_METRICS_TOTAL_SZ (8UL*301UL) -#define FD_METRICS_TILE_KIND_CNT 44 +#define FD_METRICS_TILE_KIND_CNT 43 extern const char * FD_METRICS_TILE_KIND_NAMES[FD_METRICS_TILE_KIND_CNT]; extern const ulong FD_METRICS_TILE_KIND_SIZES[FD_METRICS_TILE_KIND_CNT]; extern const fd_metrics_meta_t * FD_METRICS_TILE_KIND_METRICS[FD_METRICS_TILE_KIND_CNT]; diff --git a/src/disco/metrics/generated/fd_metrics_enums.h b/src/disco/metrics/generated/fd_metrics_enums.h index cc3e7e44494..e53fc89ca94 100644 --- a/src/disco/metrics/generated/fd_metrics_enums.h +++ b/src/disco/metrics/generated/fd_metrics_enums.h @@ -1337,16 +1337,6 @@ #define FD_METRICS_ENUM_PORT_CHECK_TCP_RESULT_V_EVICTED_NAME "evicted" #define FD_METRICS_ENUM_PORT_CHECK_TCP_RESULT_G_ALL_MASK (0x3fUL) -#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_NAME "failover_conn_reject_reason" -#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_CNT (3UL) -#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_SOURCE_IDX 0 -#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_SOURCE_NAME "source" -#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_BUDGET_IDX 1 -#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_BUDGET_NAME "budget" -#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_CAPACITY_IDX 2 -#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_V_CAPACITY_NAME "capacity" -#define FD_METRICS_ENUM_FAILOVER_CONN_REJECT_REASON_G_ALL_MASK (0x7UL) - #define FD_METRICS_ENUM_TXN_REGIME_NAME "txn_regime" #define FD_METRICS_ENUM_TXN_REGIME_CNT (3UL) #define FD_METRICS_ENUM_TXN_REGIME_V_LOAD_IDX 0 diff --git a/src/disco/metrics/generated/fd_metrics_failov.h b/src/disco/metrics/generated/fd_metrics_failov.h deleted file mode 100644 index 3adf1ca11a5..00000000000 --- a/src/disco/metrics/generated/fd_metrics_failov.h +++ /dev/null @@ -1,97 +0,0 @@ -#ifndef HEADER_fd_src_disco_metrics_generated_fd_metrics_failov_h -#define HEADER_fd_src_disco_metrics_generated_fd_metrics_failov_h - -/* THIS FILE IS GENERATED BY gen_metrics.py. DO NOT HAND EDIT. */ - -#include "../fd_metrics_base.h" -#include "fd_metrics_enums.h" - -enum { - FD_METRICS_COUNTER_FAILOV_CONN_ACCEPTED_OFF = FD_METRICS_TILE_OFF, - FD_METRICS_COUNTER_FAILOV_ACCEPT_BUDGET_EXHAUSTED_OFF, - FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_OFF, - FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_SOURCE_OFF = FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_OFF, - FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_BUDGET_OFF, - FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_CAPACITY_OFF, - FD_METRICS_COUNTER_FAILOV_CONN_RECONNECT_ADMITTED_OFF, - FD_METRICS_COUNTER_FAILOV_HANDSHAKE_TIMED_OUT_OFF, - FD_METRICS_COUNTER_FAILOV_PROOF_REJECTED_OFF, - FD_METRICS_COUNTER_FAILOV_HELLO_REJECTED_OFF, - FD_METRICS_COUNTER_FAILOV_TLS_HANDSHAKE_FAILED_OFF, - FD_METRICS_COUNTER_FAILOV_FRAME_PROTOCOL_REJECTED_OFF, - FD_METRICS_COUNTER_FAILOV_CONN_AUTHENTICATED_OFF, - FD_METRICS_COUNTER_FAILOV_HANDSHAKE_DURATION_NANOS_OFF, - FD_METRICS_GAUGE_FAILOV_HANDSHAKE_LATEST_DURATION_NANOS_OFF, - FD_METRICS_GAUGE_FAILOV_CONN_PENDING_OFF, -}; - -#define FD_METRICS_COUNTER_FAILOV_CONN_ACCEPTED_NAME "failov_conn_accepted" -#define FD_METRICS_COUNTER_FAILOV_CONN_ACCEPTED_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_CONN_ACCEPTED_DESC "TCP connections accepted before admission checks" -#define FD_METRICS_COUNTER_FAILOV_CONN_ACCEPTED_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_COUNTER_FAILOV_ACCEPT_BUDGET_EXHAUSTED_NAME "failov_accept_budget_exhausted" -#define FD_METRICS_COUNTER_FAILOV_ACCEPT_BUDGET_EXHAUSTED_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_ACCEPT_BUDGET_EXHAUSTED_DESC "Polls that stopped accepting because the syscall budget was empty" -#define FD_METRICS_COUNTER_FAILOV_ACCEPT_BUDGET_EXHAUSTED_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_NAME "failov_conn_rejected" -#define FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_DESC "Accepted TCP connections refused before candidate allocation, by reason" -#define FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_CVT (FD_METRICS_CONVERTER_NONE) -#define FD_METRICS_COUNTER_FAILOV_CONN_REJECTED_CNT (3UL) - -#define FD_METRICS_COUNTER_FAILOV_CONN_RECONNECT_ADMITTED_NAME "failov_conn_reconnect_admitted" -#define FD_METRICS_COUNTER_FAILOV_CONN_RECONNECT_ADMITTED_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_CONN_RECONNECT_ADMITTED_DESC "Candidates admitted to the reserved lane for recently authenticated addresses" -#define FD_METRICS_COUNTER_FAILOV_CONN_RECONNECT_ADMITTED_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_TIMED_OUT_NAME "failov_handshake_timed_out" -#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_TIMED_OUT_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_TIMED_OUT_DESC "Candidates closed at a phase or absolute handshake deadline" -#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_TIMED_OUT_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_COUNTER_FAILOV_PROOF_REJECTED_NAME "failov_proof_rejected" -#define FD_METRICS_COUNTER_FAILOV_PROOF_REJECTED_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_PROOF_REJECTED_DESC "Admission proofs that failed target identity authentication" -#define FD_METRICS_COUNTER_FAILOV_PROOF_REJECTED_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_COUNTER_FAILOV_HELLO_REJECTED_NAME "failov_hello_rejected" -#define FD_METRICS_COUNTER_FAILOV_HELLO_REJECTED_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_HELLO_REJECTED_DESC "Admission or TLS HELLO checks that failed" -#define FD_METRICS_COUNTER_FAILOV_HELLO_REJECTED_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_COUNTER_FAILOV_TLS_HANDSHAKE_FAILED_NAME "failov_tls_handshake_failed" -#define FD_METRICS_COUNTER_FAILOV_TLS_HANDSHAKE_FAILED_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_TLS_HANDSHAKE_FAILED_DESC "TLS handshakes that failed after admission" -#define FD_METRICS_COUNTER_FAILOV_TLS_HANDSHAKE_FAILED_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_COUNTER_FAILOV_FRAME_PROTOCOL_REJECTED_NAME "failov_frame_protocol_rejected" -#define FD_METRICS_COUNTER_FAILOV_FRAME_PROTOCOL_REJECTED_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_FRAME_PROTOCOL_REJECTED_DESC "Authenticated frames rejected by the controller" -#define FD_METRICS_COUNTER_FAILOV_FRAME_PROTOCOL_REJECTED_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_COUNTER_FAILOV_CONN_AUTHENTICATED_NAME "failov_conn_authenticated" -#define FD_METRICS_COUNTER_FAILOV_CONN_AUTHENTICATED_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_CONN_AUTHENTICATED_DESC "Completed mutual admission, TLS and HELLO authentications" -#define FD_METRICS_COUNTER_FAILOV_CONN_AUTHENTICATED_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_DURATION_NANOS_NAME "failov_handshake_duration_nanos" -#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_DURATION_NANOS_TYPE (FD_METRICS_TYPE_COUNTER) -#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_DURATION_NANOS_DESC "Cumulative time from candidate allocation to completed authentication" -#define FD_METRICS_COUNTER_FAILOV_HANDSHAKE_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_GAUGE_FAILOV_HANDSHAKE_LATEST_DURATION_NANOS_NAME "failov_handshake_latest_duration_nanos" -#define FD_METRICS_GAUGE_FAILOV_HANDSHAKE_LATEST_DURATION_NANOS_TYPE (FD_METRICS_TYPE_GAUGE) -#define FD_METRICS_GAUGE_FAILOV_HANDSHAKE_LATEST_DURATION_NANOS_DESC "Duration of the most recently completed authentication" -#define FD_METRICS_GAUGE_FAILOV_HANDSHAKE_LATEST_DURATION_NANOS_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_GAUGE_FAILOV_CONN_PENDING_NAME "failov_conn_pending" -#define FD_METRICS_GAUGE_FAILOV_CONN_PENDING_TYPE (FD_METRICS_TYPE_GAUGE) -#define FD_METRICS_GAUGE_FAILOV_CONN_PENDING_DESC "Candidates waiting for admission, TLS or HELLO authentication" -#define FD_METRICS_GAUGE_FAILOV_CONN_PENDING_CVT (FD_METRICS_CONVERTER_NONE) - -#define FD_METRICS_FAILOV_TOTAL (15UL) -extern const fd_metrics_meta_t FD_METRICS_FAILOV[FD_METRICS_FAILOV_TOTAL]; - -#endif /* HEADER_fd_src_disco_metrics_generated_fd_metrics_failov_h */ diff --git a/src/disco/metrics/generated/fd_metrics_sample_gen.c b/src/disco/metrics/generated/fd_metrics_sample_gen.c index 6f34422bcb9..aae1ea0db5c 100644 --- a/src/disco/metrics/generated/fd_metrics_sample_gen.c +++ b/src/disco/metrics/generated/fd_metrics_sample_gen.c @@ -4401,6 +4401,6 @@ fd_metrics_sample_schema_t const fd_metrics_sample_tiles[] = { { NULL, 0U, NULL, 0UL, 0UL } }; -char const * const fd_metrics_sample_skip[] = { "backt", "benchs", "failov", "solcap", NULL }; +char const * const fd_metrics_sample_skip[] = { "backt", "benchs", "solcap", NULL }; fd_metrics_sample_schema_t const fd_metrics_sample_link = { "link", 61U, fields_link, 9UL, 284UL }; diff --git a/src/disco/metrics/metrics.xml b/src/disco/metrics/metrics.xml index c012413e131..269b0fd4e9a 100644 --- a/src/disco/metrics/metrics.xml +++ b/src/disco/metrics/metrics.xml @@ -1664,28 +1664,6 @@ deliberate exception with lint-allow="") - - - - - - - - - - - - - - - - - - - - - - diff --git a/src/disco/stem/fd_stem.c b/src/disco/stem/fd_stem.c index 748fb04ec62..04156cacff2 100644 --- a/src/disco/stem/fd_stem.c +++ b/src/disco/stem/fd_stem.c @@ -169,11 +169,7 @@ AFTER_POLL_OVERRUN Is called when an overrun is detected while polling for new frags. This callback is not called when an overrun is detected in - during_frag. - - AFTER_READ_OVERRUN - Is called when an overrun is detected after during_frag. after_frag - is not called for the overwritten fragment. */ + during_frag. */ #include "../../util/log/fd_log.h" #include "../topo/fd_topo.h" @@ -821,4 +817,3 @@ STEM_(run)( fd_topo_t * topo, #undef STEM_CALLBACK_RETURNABLE_FRAG #undef STEM_CALLBACK_AFTER_FRAG #undef STEM_CALLBACK_AFTER_POLL_OVERRUN -#undef STEM_CALLBACK_AFTER_READ_OVERRUN diff --git a/src/disco/stem/fd_stem_run1.c b/src/disco/stem/fd_stem_run1.c index 0b8ea73384d..1a37258d5d7 100644 --- a/src/disco/stem/fd_stem_run1.c +++ b/src/disco/stem/fd_stem_run1.c @@ -594,9 +594,6 @@ STEM_(STEM_RUN1_NAME)( ulong in_cnt, FD_COMPILER_MFENCE(); if( FD_UNLIKELY( fd_seq_ne( seq_test, seq_found ) ) ) { /* Overrun while reading (impossible if this_in honoring our fctl) */ -#ifdef STEM_CALLBACK_AFTER_READ_OVERRUN - STEM_CALLBACK_AFTER_READ_OVERRUN( ctx ); -#endif this_in->seq = seq_test; /* Resume from here (probably reasonably current, could query in mcache sync instead) */ fd_metrics_link_in( fd_metrics_base_tl, this_in->idx )[ FD_METRICS_COUNTER_LINK_LINK_READING_OVERRUN_OFF ]++; /* No local accum since extremely rare, faster to use smaller cache line */ fd_metrics_link_in( fd_metrics_base_tl, this_in->idx )[ FD_METRICS_COUNTER_LINK_FRAG_READING_OVERRUN_OFF ] += (uint)fd_seq_diff( seq_test, seq_found ); /* No local accum since extremely rare, faster to use smaller cache line */ diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index 05bad95648c..5ee3a3377e2 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -414,7 +414,7 @@ struct fd_topo_tile { struct { char identity_key_path[ PATH_MAX ]; - int identity_key_fd; + int identity_key_fd; /* private startup pipe when the boot identity is generated */ int failover_enabled; char failover_staked_key_path[ PATH_MAX ]; ulong authorized_voter_paths_cnt; @@ -629,6 +629,7 @@ struct fd_topo_tile { struct { char identity_key_path[ PATH_MAX ]; char vote_history_path[ PATH_MAX ]; + int failover_standby; uint target_gid; uint target_uid; ulong authorized_voter_paths_cnt; diff --git a/src/discof/admin/fd_admin_tile.c b/src/discof/admin/fd_admin_tile.c index 8eb5bab1e48..bc94b9e5929 100644 --- a/src/discof/admin/fd_admin_tile.c +++ b/src/discof/admin/fd_admin_tile.c @@ -28,8 +28,8 @@ struct fd_admin_tile_ctx { int failover_enabled; /* failover hands off the identity */ int failover_switch; /* a failover identity switch is in progress */ - ulong operator_epoch; /* failover epoch, bumped by every set-identity under failover */ - int operator_pending; /* the failover tile has not been told about the last set-identity yet */ + ulong failover_epoch; /* failover epoch, bumped by every set-identity under failover */ + int failover_pending; /* the failover tile has not been told about the last set-identity yet */ uchar junk_pubkey[ 32 ]; /* the identity we booted with, the junk key under failover */ uchar failover_pubkey[ 32 ]; /* the key failover hands off, [paths.identity_key] until set-identity installs another */ @@ -688,7 +688,7 @@ failover_identity_check_poll( fd_admin_tile_ctx_t * ctx ) { static void failover_operator_begin( fd_admin_tile_ctx_t * ctx, uchar const * identity ) { - ctx->operator_epoch++; + ctx->failover_epoch++; if( !fd_memeq( identity, ctx->junk_pubkey, 32UL ) ) fd_memcpy( ctx->failover_pubkey, identity, 32UL ); FD_BASE58_ENCODE_32_BYTES( identity, new_identity ); FD_BASE58_ENCODE_32_BYTES( ctx->failover_pubkey, failover_identity ); @@ -700,8 +700,8 @@ failover_operator_begin( fd_admin_tile_ctx_t * ctx, fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, topo->tiles[ i ].id_keyswitch_obj_id ); sign->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR; } - find_identity_keyswitch( ctx, "tower" )->operator = 1UL; - ctx->operator_pending = 1; + find_identity_keyswitch( ctx, ctx->voter_name )->operator = 1UL; + ctx->failover_pending = 1; } /* Sends a pending OPERATOR once the failover link has room to spare. */ @@ -709,18 +709,18 @@ failover_operator_begin( fd_admin_tile_ctx_t * ctx, static void failover_operator_notify( fd_admin_tile_ctx_t * ctx, fd_stem_context_t * stem ) { - if( FD_LIKELY( !ctx->operator_pending ) ) return; + if( FD_LIKELY( !ctx->failover_pending ) ) return; if( FD_UNLIKELY( ctx->failov_out_idx==ULONG_MAX || stem->cr_avail[ ctx->failov_out_idx ]<4UL ) ) return; fd_failover_bus_msg_t * msg = fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk ); fd_memset( msg, 0, sizeof(*msg) ); - fd_failover_operator_t operator = { .epoch=ctx->operator_epoch }; + fd_failover_operator_t operator = { .epoch=ctx->failover_epoch }; fd_memcpy( operator.identity, ctx->identity_pubkey, 32UL ); fd_memcpy( operator.failover_identity, ctx->failover_pubkey, 32UL ); fd_memcpy( msg->payload, &operator, sizeof(operator) ); ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_OPERATOR, ctx->failov_out_chunk, sizeof(*msg), 0UL, tspub, tspub ); ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*msg), ctx->failov_out_chunk0, ctx->failov_out_wmark ); - ctx->operator_pending = 0; + ctx->failover_pending = 0; } static void FD_FN_SENSITIVE @@ -797,7 +797,7 @@ set_identity( fd_admin_tile_ctx_t * ctx, failover_operator_begin( ctx, req->keypair+32UL ); } - if( ctx->failov_out_idx!=ULONG_MAX ) find_identity_keyswitch( ctx, "tower" )->operator = 1UL; + if( ctx->failov_out_idx!=ULONG_MAX ) find_identity_keyswitch( ctx, ctx->voter_name )->operator = 1UL; ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; ulong identity_outset = (ulong)fd_log_wallclock(); @@ -1384,7 +1384,7 @@ failover_result_name( ulong result ) { case FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING: return "handoff_pending"; case FD_FAILOVER_CONTROL_RESULT_TAKEN: return "taken"; case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "staked_seen"; - case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER: return "no_final_tower"; + case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: return "no_final_history"; case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: return "replay_behind"; case FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS: return "no_active_address"; default: return "refused"; @@ -1460,8 +1460,8 @@ failover_prepare( fd_admin_tile_ctx_t * ctx, FD_TEST( result==FD_ADMINCTL_RESULT_SUCCESS ); fd_memcpy( ctx->failover_pubkey, pubkey, 32UL ); ctx->failover_enabled = 1; - ctx->operator_epoch++; - ctx->operator_pending = 1; + ctx->failover_epoch++; + ctx->failover_pending = 1; return FD_ADMINCTL_RESULT_SUCCESS; } @@ -1539,8 +1539,8 @@ failover_request( fd_admin_tile_ctx_t * ctx, /* Keep one command outstanding until the failover tile responds, even after timeout. Status reports a stalled tile as unresponsive. Publish the last set-identity OPERATOR first so the command uses its identity. */ - if( FD_UNLIKELY( ctx->failover_answered!=ctx->failover_nonce || ctx->operator_pending ) ) { - if( FD_UNLIKELY( status && ( ctx->failover_slot_idx==ULONG_MAX || ctx->operator_pending ) ) ) { + if( FD_UNLIKELY( ctx->failover_answered!=ctx->failover_nonce || ctx->failover_pending ) ) { + if( FD_UNLIKELY( status && ( ctx->failover_slot_idx==ULONG_MAX || ctx->failover_pending ) ) ) { FD_LOG_WARNING(( "`failover status` refused, the failover tile has not responded to an earlier command, check the log" )); report_admin_command_custom_result( &event, "unresponsive" ); fd_adminctl_complete( adminctl, slot_idx, FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE ); @@ -1569,7 +1569,7 @@ failover_request( fd_admin_tile_ctx_t * ctx, /* The public target and epoch must reach the controller before the command. after_credit reserves enough credits for both frames. */ failover_operator_notify( ctx, stem ); - FD_TEST( !ctx->operator_pending ); + FD_TEST( !ctx->failover_pending ); } /* Kept for the event and the log of the response. */ @@ -1630,10 +1630,10 @@ failover_switch_request( fd_admin_tile_ctx_t * ctx, fd_failover_switch_resp_t response; fd_memset( &response, 0, sizeof(response) ); response.result = FD_FAILOVER_SWITCH_ERR_DISABLED; - if( FD_UNLIKELY( ctx->failover_enabled && req.epoch!=ctx->operator_epoch ) ) { + if( FD_UNLIKELY( ctx->failover_enabled && req.epoch!=ctx->failover_epoch ) ) { FD_LOG_WARNING(( "refusing the failover tile's identity switch, set-identity ran since it asked" )); response.result = FD_FAILOVER_SWITCH_ERR_STALE; - response.operator.epoch = ctx->operator_epoch; + response.operator.epoch = ctx->failover_epoch; fd_memcpy( response.operator.identity, ctx->identity_pubkey, 32UL ); fd_memcpy( response.operator.failover_identity, ctx->failover_pubkey, 32UL ); } else if( FD_LIKELY( ctx->failover_enabled ) ) { @@ -1649,7 +1649,7 @@ failover_switch_request( fd_admin_tile_ctx_t * ctx, fd_memset( keypair, 0, 32UL ); fd_memcpy( keypair+32UL, req.identity, 32UL ); - find_identity_keyswitch( ctx, "tower" )->operator = 0UL; /* the tower keeps the failover rules */ + find_identity_keyswitch( ctx, "votor" )->operator = 0UL; /* votor enforces adoption before target participation */ ctx->failover_switch = 1; ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; ulong identity_outset = (ulong)fd_log_wallclock(); @@ -1659,12 +1659,11 @@ failover_switch_request( fd_admin_tile_ctx_t * ctx, ctx->failover_switch = 0; fd_memcpy( ctx->identity_pubkey, req.identity, 32UL ); - /* The watermark is the tower's output sequence at its halt, which - the tower tile leaves in its keyswitch result. */ - response.result = FD_FAILOVER_SWITCH_OK; - response.tower_watermark = find_identity_keyswitch( ctx, "tower" )->result; + /* Votor returns its final votor_hist sequence after halting signing. */ + response.result = FD_FAILOVER_SWITCH_OK; + response.history_watermark = find_identity_keyswitch( ctx, "votor" )->result; report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS ); - FD_LOG_NOTICE(( "failover: every tile switched to `%s`, the tower halted signing at sequence %lu", new_identity, response.tower_watermark )); + FD_LOG_NOTICE(( "failover: every tile switched to `%s`, the %s halted signing at sequence %lu", new_identity, ctx->voter_name, response.history_watermark )); } else { FD_LOG_WARNING(( "the failover tile asked for an identity switch but failover is off, refusing" )); } diff --git a/src/discof/admin/fd_adminctl.h b/src/discof/admin/fd_adminctl.h index 9c44d4471f1..47f25af930a 100644 --- a/src/discof/admin/fd_adminctl.h +++ b/src/discof/admin/fd_adminctl.h @@ -76,7 +76,7 @@ #define FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING (0x4008UL) /* the peer has not responded to our handoff */ #define FD_FAILOVER_CONTROL_RESULT_TAKEN (0x4009UL) /* the peer took our handoff, cleared once we hold the identity again or set-identity restarts failover */ #define FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN (0x400AUL) /* gossip has a fresh contact info for the staked identity from another host */ -#define FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER (0x400CUL) /* active has no eligible final vote state, see its local voting diagnostics */ +#define FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY (0x400CUL) /* active has no eligible final vote state, see its local voting diagnostics */ #define FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS (0x400EUL) /* gossip has no address for the active */ #define FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED (0x400FUL) /* no check found another holder, which does not prove the peer cannot sign */ #define FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND (0x4010UL) /* this standby's replay is too far behind the active's last vote */ @@ -183,18 +183,18 @@ FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)==24UL, failover_co #define FD_FAILOVER_ACTION_IDLE (0UL) /* no local transition is running */ #define FD_FAILOVER_ACTION_DEMOTE_SWITCH (1UL) /* waiting for the junk key to be installed */ #define FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK (2UL) /* DEMOTED sent, waiting for the peer */ -#define FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY (3UL) /* waiting for replay to reach the tower tip */ -#define FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT (4UL) /* waiting for the tower tile to adopt */ +#define FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY (3UL) /* waiting for replay to reach the finality anchor */ +#define FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT (4UL) /* waiting for votor to adopt */ #define FD_FAILOVER_ACTION_PROMOTE_SWITCH (5UL) /* waiting for the staked key to be installed */ -#define FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER (6UL) /* requesting the active's final tower */ +#define FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER (6UL) /* requesting the active's final history */ #define FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT (7UL) /* waiting for the old active to record our response */ -#define FD_FAILOVER_ACTION_DEMOTE_DRAIN (8UL) /* junk key installed, waiting for the tower to drain */ +#define FD_FAILOVER_ACTION_DEMOTE_DRAIN (8UL) /* junk key installed, waiting for votor history to drain */ #define FD_FAILOVER_ACTION_CNT (9UL) -/* The tower sources a promotion can adopt, in order of preference. */ -#define FD_FAILOVER_SOURCE_PEER (0UL) /* the tower the peer's DEMOTED gave us */ -#define FD_FAILOVER_SOURCE_VOTE_ACCOUNT (1UL) /* automatic fallback to the vote account, or forced empty history */ -#define FD_FAILOVER_SOURCE_CNT (2UL) +/* The history sources a promotion can adopt, in order of preference. */ +#define FD_FAILOVER_SOURCE_PEER (0UL) /* the history the peer's DEMOTED gave us */ +#define FD_FAILOVER_SOURCE_EMPTY (1UL) /* forced empty history with a vote bound */ +#define FD_FAILOVER_SOURCE_CNT (2UL) /* The outcome of the last handoff this machine requested or gave. */ #define FD_FAILOVER_HANDOFF_NONE (0UL) /* nothing requested or sent */ diff --git a/src/discof/admin/test_admin_tile.c b/src/discof/admin/test_admin_tile.c index 1c47a8bf0b8..077cfdc5d92 100644 --- a/src/discof/admin/test_admin_tile.c +++ b/src/discof/admin/test_admin_tile.c @@ -318,12 +318,12 @@ test_failover_identity( void ) { /* A key other than junk and staked becomes the failover identity, and nothing goes to the failover tile until every tile switched. */ failover_operator_begin( &ctx, third ); - FD_TEST( ctx.operator_epoch==1UL && fd_memeq( ctx.failover_pubkey, third, 32UL ) && ctx.operator_pending && !bus_seq ); + FD_TEST( ctx.failover_epoch==1UL && fd_memeq( ctx.failover_pubkey, third, 32UL ) && ctx.failover_pending && !bus_seq ); FD_TEST( sign_ks->param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR && tower_ks->operator==1UL ); fd_memcpy( ctx.identity_pubkey, third, 32UL ); failover_operator_notify( &ctx, bus_stem ); fd_failover_operator_t operator = bus_operator(); - FD_TEST( !ctx.operator_pending && operator.epoch==1UL ); + FD_TEST( !ctx.failover_pending && operator.epoch==1UL ); FD_TEST( fd_memeq( operator.identity, third, 32UL ) && fd_memeq( operator.failover_identity, third, 32UL ) ); /* The junk key keeps the failover identity. With three credits left @@ -332,7 +332,7 @@ test_failover_identity( void ) { failover_operator_begin( &ctx, ctx.junk_pubkey ); fd_memcpy( ctx.identity_pubkey, ctx.junk_pubkey, 32UL ); failover_operator_notify( &ctx, bus_stem ); - FD_TEST( ctx.operator_pending && bus_seq==1UL && fd_memeq( ctx.failover_pubkey, third, 32UL ) ); + FD_TEST( ctx.failover_pending && bus_seq==1UL && fd_memeq( ctx.failover_pubkey, third, 32UL ) ); FD_TEST( fd_sha512_join( fd_sha512_new( ctx.sha512 ) ) ); ctx.adminctl = fd_adminctl_join( fd_adminctl_new( adminctl_mem ) ); FD_TEST( ctx.adminctl ); @@ -353,7 +353,7 @@ test_failover_identity( void ) { bus_cr_avail = 8UL; failover_operator_notify( &ctx, bus_stem ); operator = bus_operator(); - FD_TEST( !ctx.operator_pending && operator.epoch==2UL ); + FD_TEST( !ctx.failover_pending && operator.epoch==2UL ); FD_TEST( fd_memeq( operator.identity, ctx.junk_pubkey, 32UL ) && fd_memeq( operator.failover_identity, third, 32UL ) ); /* A switch request from before set-identity gets STALE with what it diff --git a/src/discof/failover/Local.mk b/src/discof/failover/Local.mk index d748b75a66e..e9a96dff527 100644 --- a/src/discof/failover/Local.mk +++ b/src/discof/failover/Local.mk @@ -15,4 +15,6 @@ $(call make-unit-test,test_failover_channel,test_failover_channel,fd_discof fd_d $(call run-unit-test,test_failover_channel) $(call make-unit-test,test_failover_tile,test_failover_tile,fd_discof fd_disco fd_choreo fd_flamenco fd_waltz fd_tls fd_tango fd_ballet fd_util) $(call run-unit-test,test_failover_tile) +$(call make-unit-test,test_failover_tile_ag,test_failover_tile_ag,fd_discof fd_disco fd_choreo fd_flamenco fd_waltz fd_tls fd_tango fd_ballet fd_util) +$(call run-unit-test,test_failover_tile_ag) endif diff --git a/src/discof/failover/fd_failover_bus.h b/src/discof/failover/fd_failover_bus.h index c87a1a2a7bb..4ed78ebe180 100644 --- a/src/discof/failover/fd_failover_bus.h +++ b/src/discof/failover/fd_failover_bus.h @@ -48,9 +48,9 @@ typedef struct fd_failover_operator fd_failover_operator_t; /* Sent once every tile has switched, or right away when refused. */ struct fd_failover_switch_resp { - ulong result; /* FD_FAILOVER_SWITCH_* */ - ulong tower_watermark; /* the tower keyswitch result, its output sequence at the halt */ - fd_failover_operator_t operator; /* with STALE */ + ulong result; /* FD_FAILOVER_SWITCH_* */ + ulong history_watermark; /* the votor keyswitch result, its history output sequence at the halt */ + fd_failover_operator_t operator; /* with STALE */ }; typedef struct fd_failover_switch_resp fd_failover_switch_resp_t; diff --git a/src/discof/failover/fd_failover_channel.c b/src/discof/failover/fd_failover_channel.c index 7504f5ae1ba..dd1d17c178a 100644 --- a/src/discof/failover/fd_failover_channel.c +++ b/src/discof/failover/fd_failover_channel.c @@ -57,7 +57,6 @@ struct candidate { uint address; long deadline; long admission_until; - long accepted_at; int tls_started; fd_failover_hello_t local_hello; fd_failover_admission_start_t start; @@ -111,7 +110,6 @@ struct fd_failover_channel { struct candidate candidates[ FD_FAILOVER_CHANNEL_CANDIDATE_MAX ]; /* connections that are handshaking or paired */ struct source sources[ SOURCE_CNT ]; /* accept rate limit per source address */ struct known_source known_sources[ KNOWN_SOURCE_CNT ]; /* authenticated addresses, expiring */ - fd_failover_channel_metrics_t metrics; long rate_updated; /* when rate_credit was last refilled */ ulong rate_credit; /* accept syscall limit across all addresses */ long admit_updated; /* when admit_credit was last refilled */ @@ -268,9 +266,6 @@ fd_failover_channel_fini( fd_failover_channel_t * ch ) { fd_failover_tls_ctx_fini( &ch->tls_ctx ); } -fd_failover_channel_metrics_t const * -fd_failover_channel_metrics( fd_failover_channel_t const * ch ) { return &ch->metrics; } - void fd_failover_channel_set_role( fd_failover_channel_t * ch, ulong role ) { @@ -399,7 +394,6 @@ fd_failover_channel_hangup( fd_failover_channel_t * ch, void fd_failover_channel_protocol_error( fd_failover_channel_t * ch, long now ) { - ch->metrics.wire_fatal_cnt++; if( FD_LIKELY( ch->paired_idx>=0 ) ) { lose( ch, (ulong)ch->paired_idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_PROTOCOL, "it sent a message that breaks the protocol, check that both machines run the same Firedancer version" ); } @@ -514,7 +508,6 @@ start( fd_failover_channel_t * ch, c->priority = c->dialed || idx==RECONNECT_IDX; c->deadline = fd_long_sat_add( now, ch->hello_timeout ); c->admission_until = c->dialed ? c->deadline : fd_long_min( c->deadline, fd_long_sat_add( now, FD_FAILOVER_ADMISSION_START_NANOS ) ); - c->accepted_at = now; c->local_hello = ch->self_hello; c->rx_used = c->tx_used = c->tx_sent = 0UL; c->tls_started = 0; @@ -528,13 +521,12 @@ accept_candidates( fd_failover_channel_t * ch, int * busy ) { for( ulong n=0; nrate_credit = refill( ch->rate_credit, &ch->rate_updated, now, FD_FAILOVER_ACCEPT_RATE, FD_FAILOVER_ACCEPT_BURST ); - if( ch->rate_creditmetrics.accept_limited_cnt++; return; } + if( ch->rate_creditrate_credit -= FD_FAILOVER_NANOS_PER_SECOND; struct sockaddr_in addr; socklen_t len = sizeof(addr); int fd = accept4( ch->listen_fd, fd_type_pun( &addr ), &len, SOCK_NONBLOCK|SOCK_CLOEXEC ); if( fd==-1 ) return; - ch->metrics.accept_cnt++; *busy = 1; ch->admit_credit = refill( ch->admit_credit, &ch->admit_updated, now, FD_FAILOVER_ADMIT_RATE, FD_FAILOVER_ADMIT_BURST ); ch->reconnect_credit = refill( ch->reconnect_credit, &ch->reconnect_updated, now, FD_FAILOVER_RECONNECT_RATE, FD_FAILOVER_RECONNECT_BURST ); @@ -545,26 +537,22 @@ accept_candidates( fd_failover_channel_t * ch, int priority = known_source( ch, addr.sin_addr.s_addr, now ); ulong * credit = priority ? &ch->reconnect_credit : &ch->admit_credit; if( *creditmetrics.budget_reject_cnt++; close( fd ); continue; } ulong idx = priority ? RECONNECT_IDX : listen_slot( ch, now ); if( priority && ch->candidates[idx].fd!=-1 ) idx = FD_FAILOVER_CHANNEL_CANDIDATE_MAX; if( FD_UNLIKELY( idx==FD_FAILOVER_CHANNEL_CANDIDATE_MAX ) ) { - ch->metrics.capacity_reject_cnt++; close( fd ); continue; } if( !admit( ch, addr.sin_addr.s_addr, now ) ) { - ch->metrics.source_reject_cnt++; close( fd ); continue; } /* An accepted candidate leaves the session unpaired until authentication. */ if( !start( ch, idx, fd, addr.sin_addr.s_addr, now, PHASE_START_RECV ) ) { *credit -= FD_FAILOVER_NANOS_PER_SECOND; - ch->metrics.reconnect_cnt += (ulong)priority; source_admitted( ch, addr.sin_addr.s_addr, now ); ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_PEER_CONNECTED ); } @@ -628,14 +616,12 @@ hello_err_name( int err ) { case FD_FAILOVER_HELLO_ERR_BOOT_ID: return "the boot id is zero, run the same Firedancer version on both machines"; case FD_FAILOVER_HELLO_ERR_MODE: return "the consensus mode differs, run both machines in the same consensus mode"; case FD_FAILOVER_HELLO_ERR_PROOF: return "its admission proof is not signed by the failover identity, check that both machines hold the same keypair for it"; - default: return "unknown"; + default: return "unknown"; } } static void hello_refused( fd_failover_channel_t * ch, uint address, int err, long now ) { - ch->metrics.hello_reject_cnt++; - ch->metrics.proof_reject_cnt += (ulong)( err==FD_FAILOVER_HELLO_ERR_PROOF ); ulong bucket = err==HELLO_ERR_TLS_KEY ? 12UL : ( err>0 && err<=FD_FAILOVER_HELLO_ERR_PROOF ? (ulong)err : 13UL ); if( nowhello_log_at[bucket] ) return; ch->hello_log_at[bucket] = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); @@ -735,7 +721,6 @@ handshake_candidate( fd_failover_channel_t * ch, struct candidate * c = &ch->candidates[idx]; int rc = fd_failover_tls_handshake( &c->tls ); if( rc<0 ) { - ch->metrics.tls_fail_cnt++; uint reason = c->tls.conn.hs.base.reason; ulong kind = tls_log_kind( reason ); if( now>=ch->tls_log_at[kind] ) { @@ -818,9 +803,6 @@ pair_candidate( fd_failover_channel_t * ch, ch->state = fd_failover_session_step( ch->state, ch->dial_peer, FD_FAILOVER_EV_HELLO_OK ); ch->generation++; remember_source( ch, c->address, now ); - ch->metrics.paired_cnt++; - ch->metrics.handshake_latest_nanos = (ulong)fd_long_max( now-c->accepted_at, 0L ); - ch->metrics.handshake_duration_nanos += ch->metrics.handshake_latest_nanos; ch->keepalive_at = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_KEEPALIVE_NANOS ); ch->last_rx = now; ch->paired_at = now; @@ -1134,7 +1116,6 @@ fd_failover_channel_poll( fd_failover_channel_t * ch, FD_LOG_WARNING(( "the failover peer at `" FD_IP4_ADDR_FMT "` did not finish the handshake in time, check that it runs and that the machines can reach each other on [failover.listen_port]", FD_IP4_ADDR_FMT_ARGS( c->address ) )); } - ch->metrics.handshake_timeout_cnt++; /* An expired candidate closes, an outbound dial returns to backoff. */ drop( ch, i, now, FD_FAILOVER_EV_TIMEOUT ); *busy = 1; } diff --git a/src/discof/failover/fd_failover_channel.h b/src/discof/failover/fd_failover_channel.h index e449de3a873..7a5d1469970 100644 --- a/src/discof/failover/fd_failover_channel.h +++ b/src/discof/failover/fd_failover_channel.h @@ -51,24 +51,6 @@ fd_failover_clock( void ) { #define FD_FAILOVER_CHANNEL_HELLO_TIMEOUT_NANOS ( 4000000000L) #define FD_FAILOVER_CHANNEL_KEEPALIVE_NANOS (FD_FAILOVER_CHANNEL_IDLE_NANOS/4L) -struct fd_failover_channel_metrics { - ulong accept_cnt; - ulong accept_limited_cnt; - ulong source_reject_cnt; - ulong budget_reject_cnt; - ulong capacity_reject_cnt; - ulong reconnect_cnt; - ulong handshake_timeout_cnt; - ulong proof_reject_cnt; - ulong hello_reject_cnt; - ulong tls_fail_cnt; - ulong wire_fatal_cnt; - ulong paired_cnt; - ulong handshake_duration_nanos; - ulong handshake_latest_nanos; -}; -typedef struct fd_failover_channel_metrics fd_failover_channel_metrics_t; - struct fd_failover_channel; typedef struct fd_failover_channel fd_failover_channel_t; @@ -152,9 +134,6 @@ FD_FN_PURE int fd_failover_channel_listen_fd ( fd_failov FD_FN_PURE int fd_failover_channel_tx_pending( fd_failover_channel_t const * channel ); FD_FN_PURE fd_failover_hello_t const * fd_failover_channel_peer_hello( fd_failover_channel_t const * channel ); -FD_FN_PURE fd_failover_channel_metrics_t const * -fd_failover_channel_metrics( fd_failover_channel_t const * channel ); - /* Changes whenever a new authenticated session replaces the previous one. */ FD_FN_PURE ulong fd_failover_channel_generation( fd_failover_channel_t const * channel ); diff --git a/src/discof/failover/fd_failover_proto.c b/src/discof/failover/fd_failover_proto.c index e140a576cb4..5cfe48dae14 100644 --- a/src/discof/failover/fd_failover_proto.c +++ b/src/discof/failover/fd_failover_proto.c @@ -1,13 +1,13 @@ #include "fd_failover_proto.h" -#include "../../choreo/tower/fd_tower.h" -#include "../../choreo/tower/fd_tower_serdes.h" +#include "../../choreo/votor/ag_hist.h" #include "../../ballet/ed25519/fd_ed25519.h" #include "../../ballet/sha256/fd_sha256.h" #include "../../disco/keyguard/fd_keyguard.h" FD_STATIC_ASSERT( sizeof(FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN)-1UL==FD_KEYGUARD_FAILOVER_DOMAIN_SZ, client_proof_domain ); FD_STATIC_ASSERT( sizeof(FD_KEYGUARD_FAILOVER_SERVER_DOMAIN)-1UL==FD_KEYGUARD_FAILOVER_DOMAIN_SZ, server_proof_domain ); +FD_STATIC_ASSERT( AG_HIST_SER_MAX<=FD_FAILOVER_ALPENGLOW_STATE_MAX, alpenglow_state_max ); int fd_failover_hello_check( fd_failover_hello_t const * self, @@ -15,7 +15,7 @@ fd_failover_hello_check( fd_failover_hello_t const * self, if( FD_UNLIKELY( self->version!=peer->version ) ) return FD_FAILOVER_HELLO_ERR_VERSION; if( FD_UNLIKELY( self->role>FD_FAILOVER_ROLE_ACTIVE || peer->role>FD_FAILOVER_ROLE_ACTIVE ) ) return FD_FAILOVER_HELLO_ERR_ROLE; - if( FD_UNLIKELY( self->mode>=FD_FAILOVER_MODE_CNT || + if( FD_UNLIKELY( self->mode!=FD_FAILOVER_MODE_ALPENGLOW || self->mode!=peer->mode ) ) return FD_FAILOVER_HELLO_ERR_MODE; if( FD_UNLIKELY( !self->boot_id || !peer->boot_id ) ) return FD_FAILOVER_HELLO_ERR_BOOT_ID; if( FD_UNLIKELY( !fd_memeq( self->staked_pubkey, peer->staked_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_STAKED; @@ -155,13 +155,13 @@ fd_failover_demoted_encode( uchar * out, ulong last_vote_slot, uchar const * state, ulong state_sz ) { - if( FD_UNLIKELY( !state_sz || state_sz>FD_FAILOVER_TOWER_STATE_MAX ) ) return 0UL; + if( FD_UNLIKELY( !state_sz || state_sz>FD_FAILOVER_ALPENGLOW_STATE_MAX ) ) return 0UL; fd_failover_demoted_t demoted = { .handoff_id = handoff_id, .target_boot_id = target_boot_id, .last_vote_slot = last_vote_slot, - .mode = (uchar)FD_FAILOVER_MODE_TOWER, + .mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW, .state_len = (ushort)state_sz, }; fd_memcpy( out, &demoted, sizeof(demoted) ); @@ -169,6 +169,17 @@ fd_failover_demoted_encode( uchar * out, return sizeof(demoted)+state_sz; } +/* Checks that the vote history decodes exactly and ends at the slot the + DEMOTED claims. */ +static int +alpenglow_state_check( uchar const * state, + ulong state_sz, + ulong vote_slot ) { + if( FD_UNLIKELY( state_sz>FD_FAILOVER_ALPENGLOW_STATE_MAX ) ) return 0; + ag_hist_t hist[1]; + return !ag_hist_de( state, state_sz, hist ) && ag_hist_tip( hist )==vote_slot; +} + int fd_failover_demoted_decode( fd_failover_demoted_t * out, uchar const * payload, @@ -182,16 +193,10 @@ fd_failover_demoted_decode( fd_failover_demoted_t * out, ulong state_sz = payload_sz-sizeof(fd_failover_demoted_t); if( FD_UNLIKELY( (ulong)demoted.state_len!=state_sz || !state_sz || - demoted.mode!=(uchar)FD_FAILOVER_MODE_TOWER || + demoted.mode!=(uchar)FD_FAILOVER_MODE_ALPENGLOW || demoted.last_vote_slot==FD_FAILOVER_SLOT_NULL ) ) return 0; - fd_compact_tower_sync_serde_t serde; - fd_tower_vote_t votes[ FD_TOWER_VOTE_MAX ]; - ulong vote_cnt; - ulong root; - if( FD_UNLIKELY( fd_compact_tower_sync_de_exact( &serde, payload+sizeof(fd_failover_demoted_t), state_sz ) || - fd_compact_tower_sync_to_votes( &serde, votes, &vote_cnt, &root ) || - !vote_cnt || votes[ vote_cnt-1UL ].slot!=demoted.last_vote_slot ) ) return 0; + if( FD_UNLIKELY( !alpenglow_state_check( payload+sizeof(fd_failover_demoted_t), state_sz, demoted.last_vote_slot ) ) ) return 0; *out = demoted; return 1; diff --git a/src/discof/failover/fd_failover_proto.h b/src/discof/failover/fd_failover_proto.h index de08eb032be..c9849c89dea 100644 --- a/src/discof/failover/fd_failover_proto.h +++ b/src/discof/failover/fd_failover_proto.h @@ -24,8 +24,7 @@ #define FD_FAILOVER_SLOT_NULL (ULONG_MAX) /* Consensus payload formats */ -#define FD_FAILOVER_MODE_TOWER (0U) -#define FD_FAILOVER_MODE_CNT (1U) +#define FD_FAILOVER_MODE_ALPENGLOW (1U) /* Roles for each endpoint */ #define FD_FAILOVER_ROLE_STANDBY (0UL) /* Run under the junk identity. */ @@ -65,9 +64,10 @@ #define FD_FAILOVER_HELLO_ERR_MODE (9) #define FD_FAILOVER_HELLO_ERR_PROOF (10) -/* Upper bound on the consensus state payload in tower mode. A - CompactTowerSync with block id and bank hash is under 512 bytes. */ -#define FD_FAILOVER_TOWER_STATE_MAX (512UL) +/* Upper bound on the vote history payload in alpenglow mode, + AG_HIST_SER_MAX is 5274 bytes. */ +#define FD_FAILOVER_ALPENGLOW_STATE_MAX (6144UL) +#define FD_FAILOVER_STATE_MAX (FD_FAILOVER_ALPENGLOW_STATE_MAX) /* Wire protocol message bodies. Little endian, packed, fixed layout. */ struct __attribute__((packed)) fd_failover_hello { @@ -118,19 +118,19 @@ FD_STATIC_ASSERT( sizeof(fd_failover_admission_response_t)==66UL, admission_resp #define FD_FAILOVER_SERVER_FLIGHT_MAX (FD_FAILOVER_ADMISSION_MAX-sizeof(fd_failover_admission_response_t)) -/* Sent by the demoter once it runs the junk key, the final tower +/* Sent by the demoter once it runs the junk key, the final history follows it. */ struct __attribute__((packed)) fd_failover_demoted { ulong handoff_id; /* the handoff id from the standby's HANDOFF_REQUEST */ ulong target_boot_id; /* peer boot_id the handoff is for */ - ulong last_vote_slot; /* tip of the final tower */ + ulong last_vote_slot; /* tip of the final history */ uchar mode; /* FD_FAILOVER_MODE_* encoding */ - ushort state_len; /* size of the final tower that follows this struct */ + ushort state_len; /* size of the final history that follows this struct */ }; typedef struct fd_failover_demoted fd_failover_demoted_t; FD_STATIC_ASSERT( sizeof(fd_failover_demoted_t)==27UL, wire_layout ); -#define FD_FAILOVER_DEMOTED_PAYLOAD_MAX (sizeof(fd_failover_demoted_t)+FD_FAILOVER_TOWER_STATE_MAX) +#define FD_FAILOVER_DEMOTED_PAYLOAD_MAX (sizeof(fd_failover_demoted_t)+FD_FAILOVER_STATE_MAX) struct __attribute__((packed)) fd_failover_promote_ack { ulong handoff_id; @@ -235,9 +235,9 @@ fd_failover_handoff_result_decode( fd_failover_handoff_result_t * out, uchar const * payload, ulong payload_sz ); -/* Writes a DEMOTED payload, the header and then the tower, into out, +/* Writes a DEMOTED payload, the header and then the history, into out, which holds FD_FAILOVER_DEMOTED_PAYLOAD_MAX bytes. Returns the payload - size, 0 for an empty or oversized tower. */ + size, 0 for an empty or oversized history. */ ulong fd_failover_demoted_encode( uchar * out, ulong handoff_id, @@ -246,10 +246,10 @@ fd_failover_demoted_encode( uchar * out, uchar const * state, ulong state_sz ); -/* Validates a DEMOTED payload. The tower has to decode exactly and end - at last_vote_slot. Returns 1 on success, the tower is then at - payload+sizeof(fd_failover_demoted_t). Returns 0 on failure with out - left unchanged. */ +/* Validates a DEMOTED payload. The Alpenglow history must decode + exactly and end at last_vote_slot. Returns 1 on success, with the + history at payload+sizeof(fd_failover_demoted_t). Otherwise returns 0 + and leaves out unchanged. */ int fd_failover_demoted_decode( fd_failover_demoted_t * out, uchar const * payload, diff --git a/src/discof/failover/fd_failover_tile.c b/src/discof/failover/fd_failover_tile.c index 90fa00a8787..e64a5c5dc17 100644 --- a/src/discof/failover/fd_failover_tile.c +++ b/src/discof/failover/fd_failover_tile.c @@ -6,15 +6,12 @@ #include "../../flamenco/gossip/fd_gossip_message.h" #include "../../ballet/base58/fd_base58.h" #include "../../ballet/hex/fd_hex.h" -#include "../../ballet/txn/fd_txn.h" -#include "../../choreo/tower/fd_tower_serdes.h" #include "../../util/fd_version.h" #include "../../util/net/fd_ip4.h" -#include "../tower/fd_tower_tile.h" +#include "../votor/fd_votor_tile.h" #include "fd_failover_bus.h" #include "fd_failover_channel.h" -#include "../../disco/metrics/fd_metrics.h" #include #include @@ -34,10 +31,6 @@ start. */ #define FD_FAILOVER_DEADLINE_SLOTS (64UL) -/* A vote account tower short of the coverage floor is taken once replay - passes the floor by this much. */ -#define FD_FAILOVER_FLOOR_SLACK_SLOTS (512UL) - /* The promotion's replay and adopt waits also end on our clock, one second per deadline slot, so a standby whose replay froze stops. */ #define FD_FAILOVER_DEADLINE_SLOT_NANOS ((long)FD_FAILOVER_NANOS_PER_SECOND) @@ -64,15 +57,15 @@ #define FD_FAILOVER_REPLY_KEPT (1UL) /* Kept, the same DEMOTED again gets it again. */ #define FD_FAILOVER_REPLY_PENDING (2UL) /* Kept, not queued yet because the control slot was busy. */ -/* A compact tower and the slot of its last vote. */ -struct fd_failover_tower { - int valid; /* holds a tower */ - ulong tip; /* slot of its last vote */ - ulong sz; /* bytes used in state */ - uchar state[ FD_FAILOVER_TOWER_STATE_MAX ]; /* the serialized tower */ +/* A serialized vote history and its tip. */ +struct fd_failover_history { + int valid; /* holds a history */ + ulong tip; /* slot of its last vote */ + ulong sz; /* bytes used in state */ + uchar state[ FD_FAILOVER_STATE_MAX ]; /* the serialized history */ }; -typedef struct fd_failover_tower fd_failover_tower_t; +typedef struct fd_failover_history fd_failover_history_t; /* The member and boot a message or request is bound to. */ struct fd_failover_peer { @@ -134,22 +127,21 @@ typedef struct fd_failover_handoff fd_failover_handoff_t; /* The promotion in progress. */ struct fd_failover_promotion { - char label[ 48 ]; /* handoff ID or operator promotion, fixed for every step */ - ulong source; /* FD_FAILOVER_SOURCE_* we adopt from */ - int empty; /* forced recovery after the vote account could not be adopted */ - int from_peer; /* the peer's DEMOTED requested it, we send the response */ - fd_failover_peer_t peer; /* that DEMOTED's member boot, it gets our response */ - ulong handoff_id; /* and its handoff id */ - ulong floor; /* coverage floor when it started */ - int force; /* `failover promote --force`, we skip the peer checks and accept incomplete or empty history */ - int active_seen; /* an ACTIVE peer authenticated after promotion started */ - long until; /* our clock when the replay and adopt waits give up, 0 until set */ - fd_failover_tower_t adopt; /* what goes to the tower tile, empty for the vote account */ - ulong adopt_id; /* id of the adoption we asked the tower tile for */ - int retry; /* ask the tower tile again once replay passes retry_slot */ - ulong retry_slot; /* replay slot that retry waits for */ - int floor_logged; /* we said once that we wait for the floor */ - int retry_logged; /* and that we ask the tower tile again as replay moves */ + char label[ 48 ]; /* handoff ID or operator promotion, fixed for every step */ + ulong source; /* FD_FAILOVER_SOURCE_* we adopt from */ + int from_peer; /* the peer's DEMOTED requested it, we send the response */ + fd_failover_peer_t peer; /* that DEMOTED's member boot, it gets our response */ + ulong handoff_id; /* and its handoff id */ + ulong floor; /* coverage floor when it started */ + int force; /* `failover promote --force`, we skip the peer checks and accept incomplete or empty history */ + int active_seen; /* an ACTIVE peer authenticated after promotion started */ + long until; /* our clock when the replay and adopt waits give up, 0 until set */ + fd_failover_history_t adopt; /* history or the bound sent to votor */ + ulong adopt_id; /* id of the adoption we asked the votor for */ + int retry; /* ask the votor again once replay passes retry_slot */ + ulong retry_slot; /* replay slot that retry waits for */ + int floor_logged; /* we said once that we wait for the floor */ + int retry_logged; /* and that we ask the votor again as replay moves */ }; typedef struct fd_failover_promotion fd_failover_promotion_t; @@ -199,6 +191,7 @@ struct fd_failover_tile_ctx { fd_keyguard_client_t keyguard_client[ 1 ]; /* signs admission proofs and TLS */ int client_only; /* no listener and no incoming handoffs */ int input_lost; /* input overrun disables failover until restart */ + int read_pending; /* cleared only after STEM validates the fragment */ int target_pending; /* waiting for Admin to prepare the client's target */ fd_failover_hello_t hello; /* the HELLO we send */ @@ -243,7 +236,7 @@ struct fd_failover_tile_ctx { fd_failover_handoff_t handoff; /* our demotion */ int taken; /* the peer took the identity, our next tenure or a set-identity clears this */ - fd_failover_tower_t peer_tower; /* stored peer tower, from a DEMOTED */ + fd_failover_history_t peer_history; /* stored peer history, from a DEMOTED */ fd_failover_promotion_t promotion; /* the promotion in progress */ fd_failover_reply_t reply; /* our last response to a DEMOTED */ @@ -266,7 +259,7 @@ struct fd_failover_tile_ctx { fd_failover_id_switch_t id_switch; /* the identity switch we asked the admin tile for */ - /* Adoption request to the tower tile and its response. */ + /* Adoption request to the votor and its response. */ ulong adopt_out_idx; /* adoption request out link */ fd_wksp_t * adopt_out_mem; /* its workspace */ ulong adopt_out_chunk0; /* its first chunk */ @@ -277,26 +270,31 @@ struct fd_failover_tile_ctx { fd_wksp_t * adopt_in_mem; /* its workspace */ ulong adopt_in_chunk0; /* its first chunk */ ulong adopt_in_wmark; /* its last chunk */ - fd_tower_adopt_result_t adopt_result; /* the adoption response copied in during_frag */ + fd_votor_adopt_result_t adopt_result; /* the adoption response copied in during_frag */ ulong adopt_result_id; /* the request it answers */ int adopt_result_fresh; /* adopt_result arrived and is not handled yet */ - /* tower_out, reliable vote history updates */ - ulong tower_in_idx; /* tower_out in link */ - fd_wksp_t * tower_in_mem; /* its workspace */ - ulong tower_in_chunk0; /* its first chunk */ - ulong tower_in_wmark; /* its last chunk */ + /* Unreliable vote history input. Any overrun disables failover. */ + ulong hist_in_idx; /* votor_hist in link */ + fd_wksp_t * hist_in_mem; /* its workspace */ + ulong hist_in_chunk0; /* its first chunk */ + ulong hist_in_wmark; /* its last chunk */ + + int hist_fresh; /* a history frame arrived and is not handled yet */ + ulong hist_seen_seq; /* seq of the last votor_hist frag taken in, ULONG_MAX before any */ + int hist_gap; /* a votor_hist frag was skipped since the cached history was built */ - fd_tower_slot_done_t slot_done; /* the tower_out frame copied in during_frag */ - int slot_done_fresh; /* slot_done arrived and is not handled yet */ - ulong tower_seen_seq; /* seq of the last tower_out frag taken in, ULONG_MAX before any */ - int tower_gap; /* a tower_out frag was skipped since the cached tower was built */ + /* Votor history and adoption bounds. */ + fd_votor_hist_msg_t hist; /* the votor_hist frame copied in during_frag */ + ulong adopt_anchor; /* finality anchor of the history being adopted, SLOT_NULL when none */ + ulong empty_vote_after; /* an empty history votes only past this slot, SLOT_NULL until known */ + int hist_warned; /* we warned of a bad frame and no good one came since */ ulong replay_slot; /* latest replayed slot */ ulong root_slot; /* latest root */ ulong last_vote_slot; /* our latest vote */ - fd_failover_tower_t current_tower; /* tower of our latest vote */ + fd_failover_history_t current_history; /* history of our latest vote */ uchar rx[ FD_FAILOVER_PAYLOAD_MAX ]; /* the control message being decoded */ }; @@ -350,7 +348,7 @@ privileged_init( fd_topo_t const * topo, ctx->replay_slot = FD_FAILOVER_SLOT_NULL; ctx->root_slot = FD_FAILOVER_SLOT_NULL; ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; - ctx->tower_seen_seq = ULONG_MAX; + ctx->hist_seen_seq = ULONG_MAX; FD_TEST( fd_rng_secure( &ctx->handoff_base, sizeof(ulong) ) ); /* The boot identity goes into HELLO and TLS. With failover enabled @@ -371,7 +369,7 @@ privileged_init( fd_topo_t const * topo, } /* The vote key is a base58 pubkey or a keypair file, same as in the - tower tile. */ + votor. */ if( FD_UNLIKELY( !fd_base58_decode_32( tile->failov.vote_account_path, ctx->hello.vote_account ) ) ) { if( FD_UNLIKELY( !strcmp( tile->failov.vote_account_path, "" ) ) ) FD_LOG_ERR(( "missing [paths.vote_account]" )); uchar const * vote_account = fd_keyload_load( tile->failov.vote_account_path, 1 ); @@ -379,11 +377,14 @@ privileged_init( fd_topo_t const * topo, fd_keyload_unload( vote_account, 1 ); } + ctx->adopt_anchor = FD_FAILOVER_SLOT_NULL; + ctx->empty_vote_after = FD_FAILOVER_SLOT_NULL; + /* Every boot starts passively, as a standby on the junk key. */ ctx->role = FD_FAILOVER_ROLE_STANDBY; ctx->hello.version = (ushort)FD_FAILOVER_VERSION; ctx->hello.role = (uchar)ctx->role; - ctx->hello.mode = (uchar)FD_FAILOVER_MODE_TOWER; + ctx->hello.mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; while( FD_UNLIKELY( !ctx->hello.boot_id ) ) FD_TEST( fd_rng_secure( &ctx->hello.boot_id, sizeof(ulong) ) ); /* The commit field has the hash bytes, a build without one leaves it zero. */ @@ -428,10 +429,10 @@ unprivileged_init( fd_topo_t const * topo, fd_failover_tile_ctx_t * ctx = fd_topo_obj_laddr( topo, tile->tile_obj_id ); ctx->gossip_in_idx = ULONG_MAX; - ctx->tower_in_idx = ULONG_MAX; + ctx->hist_in_idx = ULONG_MAX; ctx->admin_in_idx = ULONG_MAX; ctx->adopt_in_idx = ULONG_MAX; - ctx->adopt_out_idx = fd_topo_find_tile_out_link( topo, tile, "adopt_tower", 0UL ); + ctx->adopt_out_idx = fd_topo_find_tile_out_link( topo, tile, "failov_votor", 0UL ); if( FD_LIKELY( ctx->adopt_out_idx!=ULONG_MAX ) ) { fd_topo_link_t const * link = &topo->links[ tile->out_link_id[ ctx->adopt_out_idx ] ]; ctx->adopt_out_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; @@ -457,7 +458,7 @@ unprivileged_init( fd_topo_t const * topo, ctx->gossip_in_mtu = link->mtu; continue; } - if( FD_LIKELY( !strcmp( link->name, "tower_adopt" ) ) ) { + if( FD_LIKELY( !strcmp( link->name, "votor_failov" ) ) ) { ctx->adopt_in_idx = i; ctx->adopt_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; ctx->adopt_in_chunk0 = fd_dcache_compact_chunk0( ctx->adopt_in_mem, link->dcache ); @@ -471,11 +472,11 @@ unprivileged_init( fd_topo_t const * topo, ctx->admin_in_wmark = fd_dcache_compact_wmark ( ctx->admin_in_mem, link->dcache, link->mtu ); continue; } - if( FD_LIKELY( !strcmp( link->name, "tower_out" ) ) ) { - ctx->tower_in_idx = i; - ctx->tower_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; - ctx->tower_in_chunk0 = fd_dcache_compact_chunk0( ctx->tower_in_mem, link->dcache ); - ctx->tower_in_wmark = fd_dcache_compact_wmark ( ctx->tower_in_mem, link->dcache, link->mtu ); + if( FD_LIKELY( !strcmp( link->name, "votor_hist" ) ) ) { + ctx->hist_in_idx = i; + ctx->hist_in_mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; + ctx->hist_in_chunk0 = fd_dcache_compact_chunk0( ctx->hist_in_mem, link->dcache ); + ctx->hist_in_wmark = fd_dcache_compact_wmark ( ctx->hist_in_mem, link->dcache, link->mtu ); continue; } /* The keyguard client reads sign_failov itself. */ @@ -528,10 +529,10 @@ sync_session( fd_failover_tile_ctx_t * ctx ) { if( FD_UNLIKELY( ( ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY || ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ) && peer->role==(uchar)FD_FAILOVER_ROLE_ACTIVE ) ) ctx->promotion.active_seen = 1; - if( FD_UNLIKELY( peer->role==(uchar)FD_FAILOVER_ROLE_ACTIVE && ( ctx->peer_tower.valid ) ) ) + if( FD_UNLIKELY( peer->role==(uchar)FD_FAILOVER_ROLE_ACTIVE && ( ctx->peer_history.valid ) ) ) FD_LOG_NOTICE(( "peer boot %016lx authenticated ACTIVE, discarding saved final state from before its voting tenure, keeping coverage floors", peer->boot_id )); if( FD_UNLIKELY( peer->role==(uchar)FD_FAILOVER_ROLE_ACTIVE ) ) { - ctx->peer_tower.valid = 0; + ctx->peer_history.valid = 0; } } } @@ -578,24 +579,13 @@ deadline_expired( fd_failover_tile_ctx_t const * ctx ) { ctx->replay_slot>ctx->deadline_slot; } -/* Sets the promotion deadline, in slots and on our clock, with room for - the floor slack when adopting the vote account. */ +/* Sets the promotion deadline in slots and on the monotonic clock. */ static void promote_deadline_arm( fd_failover_tile_ctx_t * ctx, long now ) { deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); - if( FD_UNLIKELY( ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT && - ctx->promotion.floor!=FD_FAILOVER_SLOT_NULL && - ctx->deadline_slot!=FD_FAILOVER_SLOT_NULL ) ) { - ctx->deadline_slot = fd_ulong_max( ctx->deadline_slot, - fd_ulong_sat_add( ctx->promotion.floor, FD_FAILOVER_FLOOR_SLACK_SLOTS+FD_FAILOVER_DEADLINE_SLOTS ) ); - } - if( FD_UNLIKELY( !ctx->promotion.until ) ) { - ulong slots = FD_FAILOVER_DEADLINE_SLOTS; - if( FD_UNLIKELY( ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT && - ctx->promotion.floor!=FD_FAILOVER_SLOT_NULL ) ) slots += FD_FAILOVER_FLOOR_SLACK_SLOTS; - ctx->promotion.until = fd_long_sat_add( now, (long)slots*FD_FAILOVER_DEADLINE_SLOT_NANOS ); - } + if( FD_UNLIKELY( !ctx->promotion.until ) ) + ctx->promotion.until = fd_long_sat_add( now, (long)FD_FAILOVER_DEADLINE_SLOTS*FD_FAILOVER_DEADLINE_SLOT_NANOS ); } /* Returns whether the promotion passed its slot deadline or its clock @@ -711,9 +701,9 @@ relayed_refusal( ulong result, case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: *hint = "the machine we dialed is not the active any more, check `failover status` on both machines"; return "BAD_ROLE"; - case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER: + case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: *hint = "the active has no eligible final vote state to hand over and keeps the identity, its log says why"; - return "NO_FINAL_TOWER"; + return "NO_FINAL_HISTORY"; case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: *hint = "this machine's replay is too far behind the active's last vote and the active keeps the identity, run `failover promote` here again once this machine has caught up"; return "REPLAY_BEHIND"; @@ -776,86 +766,63 @@ pending_flush( fd_failover_tile_ctx_t * ctx, ctx->tx.valid = 0; } -/* Keep the tower of our latest vote for the failover identity. - It is the final tower we hand over at a demotion. A malformed - transaction is logged and skipped, the previous tower stays. */ static void -prepare_consensus( fd_failover_tile_ctx_t * ctx, - fd_tower_slot_done_t const * done ) { - if( FD_UNLIKELY( ctx->target_pending ) ) return; - - uchar txn_mem[ FD_TXN_MAX_SZ ] __attribute__((aligned(alignof(fd_txn_t)))); - int parsed = done->vote_txn_sz && done->vote_txn_sz<=sizeof(done->vote_txn) && - fd_txn_parse( done->vote_txn, done->vote_txn_sz, txn_mem, NULL ); - /* The fee payer is the node identity, even with a separate vote authority. - A client can vote under its boot key until it adopts the target history. - Check the transaction identity because queued votes can outlive a switch. */ - if( FD_LIKELY( parsed ) ) { - fd_acct_addr_t const * accounts = fd_txn_get_acct_addrs( (fd_txn_t const *)txn_mem, done->vote_txn ); - if( FD_UNLIKELY( !fd_memeq( accounts[ 0 ].b, ctx->hello.staked_pubkey, 32UL ) ) ) return; - } - /* An unreadable transaction still raises the floor conservatively. */ - ctx->last_vote_slot = done->vote_slot; - if( FD_LIKELY( ctx->own_floor==FD_FAILOVER_SLOT_NULL || done->vote_slot>ctx->own_floor ) ) ctx->own_floor = done->vote_slot; - - fd_compact_tower_sync_serde_t serde; - if( FD_UNLIKELY( !parsed || - !fd_txn_parse_simple_vote( (fd_txn_t const *)txn_mem, done->vote_txn, &serde ) ) ) { - FD_LOG_WARNING(( "tower produced an invalid vote transaction" )); - return; - } - - fd_tower_vote_t votes[ FD_TOWER_VOTE_MAX ]; - ulong vote_cnt; - ulong root; - if( FD_UNLIKELY( fd_compact_tower_sync_to_votes( &serde, votes, &vote_cnt, &root ) || - !vote_cnt || votes[ vote_cnt-1UL ].slot!=done->vote_slot ) ) { - FD_LOG_WARNING(( "tower vote transaction does not match its slot metadata" )); - return; - } - - uchar state[ FD_FAILOVER_TOWER_STATE_MAX ]; +prepare_history( fd_failover_tile_ctx_t * ctx, + fd_votor_hist_msg_t const * hist ) { + uchar state[ FD_FAILOVER_ALPENGLOW_STATE_MAX ]; ulong state_sz = 0UL; - if( FD_UNLIKELY( fd_compact_tower_sync_ser( &serde, state, sizeof(state), &state_sz ) ) ) { - FD_LOG_WARNING(( "tower vote transaction exceeds the failover state limit" )); + if( FD_UNLIKELY( ag_hist_ser( &hist->hist, state, FD_FAILOVER_ALPENGLOW_STATE_MAX, &state_sz ) || + ag_hist_tip( &hist->hist )!=hist->vote_slot ) ) { + /* Warn once until a good frame arrives. */ + if( FD_UNLIKELY( !ctx->hist_warned ) ) FD_LOG_WARNING(( "votor produced a vote history that does not match its slot metadata, until a good one arrives a handoff hands over nothing" )); + ctx->hist_warned = 1; return; } - - fd_memcpy( ctx->current_tower.state, state, state_sz ); - ctx->current_tower.valid = 1; - ctx->current_tower.tip = done->vote_slot; - ctx->current_tower.sz = state_sz; - ctx->tower_gap = 0; /* anything skipped before this vote is superseded by it */ + ctx->hist_warned = 0; + fd_memcpy( ctx->current_history.state, state, state_sz ); + ctx->current_history.valid = 1; + ctx->current_history.tip = hist->vote_slot; + ctx->current_history.sz = state_sz; + ctx->hist_gap = 0; /* a complete snapshot supersedes whatever was skipped */ } -/* Fold one tower slot_done into the local slot view. The tower can - report slots out of order across forks, so the replay slot is a - running maximum. */ +/* consume_slot_done for Alpenglow, it takes in a votor_hist frame. */ static void -consume_slot_done( fd_failover_tile_ctx_t * ctx, - fd_tower_slot_done_t const * done ) { - if( FD_LIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || done->replay_slot>ctx->replay_slot ) ) - ctx->replay_slot = done->replay_slot; - if( FD_LIKELY( done->root_slot!=FD_FAILOVER_SLOT_NULL ) ) ctx->root_slot = done->root_slot; - if( FD_LIKELY( done->has_vote_txn && done->vote_slot!=FD_FAILOVER_SLOT_NULL ) ) { - prepare_consensus( ctx, done ); - } -} - -/* Sends the tower we adopt to the tower tile, empty for the vote - account. The response comes back with the request id so we can tell a - late one apart. Returns the id, or ULONG_MAX if there was no request to send. */ +consume_hist( fd_failover_tile_ctx_t * ctx, + fd_votor_hist_msg_t const * hist ) { + if( FD_LIKELY( hist->replay_slot!=FD_FAILOVER_SLOT_NULL && + ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || hist->replay_slot>ctx->replay_slot ) ) ) + ctx->replay_slot = hist->replay_slot; + if( FD_LIKELY( hist->root_slot!=FD_FAILOVER_SLOT_NULL ) ) ctx->root_slot = hist->root_slot; + /* Boot-key histories can arrive during preparation or after the switch. */ + if( FD_UNLIKELY( ctx->target_pending || !fd_memeq( hist->identity.uc, ctx->hello.staked_pubkey, 32UL ) ) ) return; + int producing = ctx->role==FD_FAILOVER_ROLE_ACTIVE || ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH; + if( FD_LIKELY( hist->has_vote && hist->vote_slot!=FD_FAILOVER_SLOT_NULL ) ) { + /* Every vote here is one we signed, so a later promotion has to cover it. */ + ctx->last_vote_slot = hist->vote_slot; + if( FD_LIKELY( ctx->own_floor==FD_FAILOVER_SLOT_NULL || hist->vote_slot>ctx->own_floor ) ) ctx->own_floor = hist->vote_slot; + prepare_history( ctx, hist ); + } else if( FD_UNLIKELY( producing && ctx->current_history.valid && hist->vote_slot!=FD_FAILOVER_SLOT_NULL && hist->vote_slot>=ctx->last_vote_slot ) ) { + /* A frame with no new vote can still change the history, after a LEADER or + a vote marked unsent. */ + ctx->last_vote_slot = hist->vote_slot; + prepare_history( ctx, hist ); + } +} + +/* Sends history or a forced empty-history bound to votor. The response + carries the request ID to distinguish late replies. Returns that ID, + or ULONG_MAX if there was no request to send. */ static ulong publish_adopt_state( fd_failover_tile_ctx_t * ctx, fd_stem_context_t * stem ) { if( FD_UNLIKELY( ctx->adopt_out_idx==ULONG_MAX || - ( !ctx->promotion.adopt.sz && ctx->promotion.source!=FD_FAILOVER_SOURCE_VOTE_ACCOUNT ) ) ) return ULONG_MAX; + !ctx->promotion.adopt.sz ) ) return ULONG_MAX; if( FD_UNLIKELY( !++ctx->adopt_request_id ) ) ctx->adopt_request_id++; fd_memcpy( fd_chunk_to_laddr( ctx->adopt_out_mem, ctx->adopt_out_chunk ), ctx->promotion.adopt.state, ctx->promotion.adopt.sz ); ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); - ulong ctl = ctx->promotion.empty && ctx->promotion.force && !ctx->promotion.from_peer ? FD_TOWER_ADOPT_CTL_EMPTY : 0UL; - fd_stem_publish( stem, ctx->adopt_out_idx, ctx->adopt_request_id, ctx->adopt_out_chunk, ctx->promotion.adopt.sz, ctl, tspub, tspub ); + fd_stem_publish( stem, ctx->adopt_out_idx, ctx->adopt_request_id, ctx->adopt_out_chunk, ctx->promotion.adopt.sz, 0UL, tspub, tspub ); ctx->adopt_out_chunk = fd_dcache_compact_next( ctx->adopt_out_chunk, ctx->promotion.adopt.sz, ctx->adopt_out_chunk0, ctx->adopt_out_wmark ); ctx->adopt_result_fresh = 0; return ctx->adopt_request_id; @@ -933,10 +900,10 @@ operator_switched( fd_failover_tile_ctx_t * ctx, fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) ); fd_memset( &ctx->reply, 0, sizeof(ctx->reply) ); fd_memset( &ctx->tx, 0, sizeof(ctx->tx) ); - ctx->peer_tower.valid = 0; - ctx->current_tower.valid = 0; + ctx->peer_history.valid = 0; + ctx->current_history.valid = 0; ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; - ctx->tower_gap = 0; + ctx->hist_gap = 0; ctx->taken = 0; ctx->last_requested = 0; ctx->cmd_addr = 0U; @@ -1023,34 +990,42 @@ start_demotion( fd_failover_tile_ctx_t * ctx, ctx->action = FD_FAILOVER_ACTION_DEMOTE_SWITCH; } -/* The final tower has to be the tower of our last vote, with no tower +/* Returns the slot a standby's replay has to reach before it adopts what + we hand over. */ +static ulong +ready_slot( fd_failover_tile_ctx_t const * ctx ) { + ag_hist_t hist[1]; + if( FD_UNLIKELY( ag_hist_de( ctx->current_history.state, ctx->current_history.sz, hist ) ) ) return ctx->last_vote_slot; + return hist->anchor; +} + +/* The final history has to be the history of our last vote, with no history frag skipped since it was built. */ static int -final_tower_ok( fd_failover_tile_ctx_t const * ctx ) { - return ctx->current_tower.valid && ctx->current_tower.sz && +final_history_ok( fd_failover_tile_ctx_t const * ctx ) { + return ctx->current_history.valid && ctx->current_history.sz && ctx->last_vote_slot!=FD_FAILOVER_SLOT_NULL && - ctx->current_tower.tip==ctx->last_vote_slot && !ctx->tower_gap; + ctx->current_history.tip==ctx->last_vote_slot && !ctx->hist_gap; } -/* Called once the junk key is installed everywhere. Our latest tower is +/* Called once the junk key is installed everywhere. Our latest history is the final one, we hand it to the peer. */ static void demotion_switched( fd_failover_tile_ctx_t * ctx ) { - /* The junk key is installed and the Tower has drained. */ + /* The junk key is installed and votor has drained. */ set_role( ctx, FD_FAILOVER_ROLE_STANDBY ); - FD_LOG_NOTICE(( "handoff %lu: the tower drained, we are a standby now", ctx->handoff.id )); + FD_LOG_NOTICE(( "handoff %lu: the vote history stream drained, we are a standby now", ctx->handoff.id )); - int final_ok = final_tower_ok( ctx ); + int final_ok = final_history_ok( ctx ); - ctx->handoff.payload_sz = final_ok - ? fd_failover_demoted_encode( ctx->handoff.payload, ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot, - ctx->current_tower.state, ctx->current_tower.sz ) - : 0UL; + ctx->handoff.payload_sz = !final_ok ? 0UL : + fd_failover_demoted_encode( ctx->handoff.payload, ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot, + ctx->current_history.state, ctx->current_history.sz ); if( FD_UNLIKELY( !ctx->handoff.payload_sz ) ) { - /* We have no final tower to hand over, or the one we have is not the - tower of our last vote. The peer gets no final state and cannot + /* We have no final history to hand over, or the one we have is not the + history of our last vote. The peer gets no final state and cannot promote. */ - FD_LOG_WARNING(( "handoff %lu has no final tower for its last vote %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->last_vote_slot )); + FD_LOG_WARNING(( "handoff %lu has no final vote history for its last vote %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->last_vote_slot )); /* End demotion without sending final state. */ demotion_abort( ctx ); return; @@ -1078,11 +1053,12 @@ reject_name( uint reason ) { static char const * adopt_err_name( ulong result ) { switch( result ) { - case FD_TOWER_ADOPT_ERR_DECODE: return "decode error"; - case FD_TOWER_ADOPT_ERR_INVALID: return "invalid"; - case FD_TOWER_ADOPT_ERR_UNREPLAYED_ROOT: return "root not replayed"; - case FD_TOWER_ADOPT_ERR_BLOCK_MISMATCH: return "block mismatch"; - case FD_TOWER_ADOPT_ERR_UNREPLAYED: return "votes not replayed"; + case FD_VOTOR_ADOPT_ERR_DECODE: return "decode error"; + case FD_VOTOR_ADOPT_ERR_INVALID: return "invalid"; + case FD_VOTOR_ADOPT_ERR_UNREPLAYED_ROOT: return "finality anchor not replayed"; + case FD_VOTOR_ADOPT_ERR_UNREPLAYED: return "consensus not initialized"; + case FD_VOTOR_ADOPT_ERR_STALE: return "older than the votes this machine sent as this identity"; + case FD_VOTOR_ADOPT_ERR_CAPACITY: return "insufficient slot state capacity"; default: return "unknown"; } } @@ -1090,9 +1066,9 @@ adopt_err_name( ulong result ) { static char const * source_name( ulong source ) { switch( source ) { - case FD_FAILOVER_SOURCE_PEER: return "the tower the peer gave us"; - case FD_FAILOVER_SOURCE_VOTE_ACCOUNT: return "the vote account"; - default: return "unknown"; + case FD_FAILOVER_SOURCE_PEER: return "the vote history the peer gave us"; + case FD_FAILOVER_SOURCE_EMPTY: return "an empty history"; + default: return "unknown"; } } @@ -1141,9 +1117,9 @@ control_refusal( ulong result, case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: *hint = "gossip showed the staked identity at another host within 15 seconds, stop it or wait, or `failover promote --force` if it cannot sign"; return "STAKED_SEEN"; - case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER: + case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: *hint = "the active has no eligible final vote state to hand over and keeps the identity, its log explains whether no vote is known or history is missing, check its voting progress before retrying"; - return "NO_FINAL_TOWER"; + return "NO_FINAL_HISTORY"; default: *hint = "the failover tile does not know this command"; return "UNSUPPORTED"; @@ -1162,14 +1138,14 @@ request_wait_result( fd_failover_tile_ctx_t * ctx ) { } /* Stand back down. If the peer's DEMOTED asked for the promotion, we - tell it why and drop the tower it handed us, because after a refusal + tell it why and drop the history it handed us, because after a refusal either machine may recover on its own without reporting its later votes. */ static void reject_promotion( fd_failover_tile_ctx_t * ctx, uchar reason, - int bad_tower ) { - if( FD_UNLIKELY( bad_tower && ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ) ctx->peer_tower.valid = 0; + int bad_history ) { + if( FD_UNLIKELY( bad_history && ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ) ctx->peer_history.valid = 0; ctx->stuck = 1; /* Failed local promotion returns to idle on the standby. */ ctx->action = FD_FAILOVER_ACTION_IDLE; @@ -1178,7 +1154,7 @@ reject_promotion( fd_failover_tile_ctx_t * ctx, "run `failover promote --force` again once this machine has caught up and the other cannot sign", ctx->promotion.label, reject_name( reason ) )); return; } - ctx->peer_tower.valid = 0; + ctx->peer_history.valid = 0; ctx->promotion.from_peer = 0; /* A failed handoff waits for the old active to confirm our refusal. */ request_wait_result( ctx ); @@ -1201,7 +1177,53 @@ coverage_floor( fd_failover_tile_ctx_t const * ctx ) { return fd_ulong_max( ctx->peer_floor, ctx->own_floor ); } -/* Take the staked identity with the tower from source. A peer is given +/* Records the finality anchor of the saved history we adopt, which replay + has to reach first. */ +static void +promote_start( fd_failover_tile_ctx_t * ctx ) { + ctx->adopt_anchor = FD_FAILOVER_SLOT_NULL; + ctx->empty_vote_after = FD_FAILOVER_SLOT_NULL; + if( FD_LIKELY( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) { + ag_hist_t hist[1]; + if( FD_LIKELY( !ag_hist_de( ctx->promotion.adopt.state, ctx->promotion.adopt.sz, hist ) ) ) ctx->adopt_anchor = hist->anchor; + } +} + +static int +replay_ready( fd_failover_tile_ctx_t * ctx ) { + if( FD_LIKELY( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) + return ctx->adopt_anchor==FD_FAILOVER_SLOT_NULL || ctx->replay_slot>=ctx->adopt_anchor; + ulong floor = ctx->promotion.floor; + ctx->empty_vote_after = floor==FD_FAILOVER_SLOT_NULL ? ctx->replay_slot : fd_ulong_max( ctx->replay_slot, floor ); + if( FD_UNLIKELY( floor==FD_FAILOVER_SLOT_NULL ) ) { + FD_LOG_WARNING(( "%s: --force accepts empty Alpenglow history, no vote or leader window at or below replay slot %lu, no earlier vote as the staked identity " + "is known, so a vote the peer cast above that slot is not covered, local vote marks and stronger local bounds remain", + ctx->promotion.label, ctx->empty_vote_after )); + } else { + FD_LOG_WARNING(( "%s: --force accepts empty Alpenglow history, no vote or leader window at or below slot %lu, which covers replay at slot %lu and the " + "last vote known as the staked identity at slot %lu, local vote marks and stronger local bounds remain", + ctx->promotion.label, ctx->empty_vote_after, ctx->replay_slot, floor )); + } + FD_STORE( ulong, ctx->promotion.adopt.state, ctx->empty_vote_after ); + ctx->promotion.adopt.sz = FD_VOTOR_ADOPT_EMPTY_SZ; + return 1; +} + +/* Logs what the votor adopted. */ +static void +log_adopted( fd_failover_tile_ctx_t const * ctx ) { + ulong bound = ctx->adopt_result.vote_bound; + if( FD_UNLIKELY( ctx->promotion.source==FD_FAILOVER_SOURCE_EMPTY ) ) { + FD_LOG_NOTICE(( "%s: the votor started an empty vote history, no vote at or below slot %lu", ctx->promotion.label, bound )); + } else if( FD_LIKELY( bound==FD_FAILOVER_SLOT_NULL ) ) { + FD_LOG_NOTICE(( "%s: the votor adopted the vote history ending at slot %lu, finality anchor at slot %lu", ctx->promotion.label, ctx->adopt_result.vote_slot, ctx->adopt_anchor )); + } else { + FD_LOG_NOTICE(( "%s: the votor adopted the vote history ending at slot %lu, finality anchor at slot %lu, no vote at or below slot %lu", ctx->promotion.label, + ctx->adopt_result.vote_slot, ctx->adopt_anchor, bound )); + } +} + +/* Take the staked identity with the history from source. A peer is given when that member's DEMOTED with handoff_id asked for it, it gets our response. An operator promotion passes NULL and 0. force is promote --force. */ @@ -1215,7 +1237,7 @@ start_promotion( fd_failover_tile_ctx_t * ctx, fd_failover_peer_t bound = {0}; if( from_peer ) bound = *peer; fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) ); - if( source==FD_FAILOVER_SOURCE_PEER ) ctx->promotion.adopt = ctx->peer_tower; + if( source==FD_FAILOVER_SOURCE_PEER ) ctx->promotion.adopt = ctx->peer_history; else ctx->promotion.adopt.tip = FD_FAILOVER_SLOT_NULL; if( from_peer ) { ctx->promotion.peer = bound; @@ -1228,65 +1250,60 @@ start_promotion( fd_failover_tile_ctx_t * ctx, ctx->promotion.from_peer = from_peer; ctx->promotion.force = force; ctx->promotion.floor = coverage_floor( ctx ); - /* Until this tenure votes, its final tower is exactly the adopted one, - not a tower cached from an earlier tenure. */ - ctx->current_tower = ctx->promotion.adopt; - ctx->tower_gap = 0; - ctx->last_vote_slot = ctx->promotion.adopt.tip; - ctx->stuck = 0; + /* Until this tenure votes, its final history is exactly the adopted one, + not a history cached from an earlier tenure. */ + ctx->current_history = ctx->promotion.adopt; + ctx->hist_gap = 0; + ctx->last_vote_slot = ctx->promotion.adopt.tip; + ctx->stuck = 0; + promote_start( ctx ); deadline_start( ctx, FD_FAILOVER_DEADLINE_SLOTS ); /* Replay must reach the selected history before adoption. */ ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY; - if( FD_UNLIKELY( from_peer && ctx->promotion.adopt.tip!=FD_FAILOVER_SLOT_NULL && - ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || ctx->replay_slotpromotion.adopt.tip ) ) ) { - FD_LOG_NOTICE(( "%s: the promotion waits for replay to reach slot %lu before it adopts %s", ctx->promotion.label, ctx->promotion.adopt.tip, source_name( source ) )); + /* A vote history waits for replay to reach its finality anchor. */ + if( FD_UNLIKELY( from_peer && ctx->adopt_anchor!=FD_FAILOVER_SLOT_NULL && + ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || ctx->replay_slotadopt_anchor ) ) ) { + FD_LOG_NOTICE(( "%s: the promotion waits for replay to reach the finality anchor at slot %lu before it adopts %s", ctx->promotion.label, ctx->adopt_anchor, source_name( source ) )); } } -/* An operator promotion tries the stored peer tower, then the vote - account, then with --force an empty tower. We enter only before the - adoption request goes out or after its response, so we do not overlap - an adoption. A handoff adopts its bound final state or refuses. */ +/* Forced promotion tries saved peer history, then bounded empty history. + Enter only before adoption is sent or after its response. A planned + handoff adopts its bound final state or refuses. */ static int promote_fallback( fd_failover_tile_ctx_t * ctx, char const * reason ) { - if( FD_UNLIKELY( ctx->promotion.from_peer || ctx->promotion.empty ) ) return 0; - if( ctx->promotion.source!=FD_FAILOVER_SOURCE_VOTE_ACCOUNT ) { - FD_LOG_NOTICE(( "%s: %s is not eligible (%s), trying the vote account", ctx->promotion.label, - source_name( ctx->promotion.source ), reason )); - if( ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ctx->peer_tower.valid = 0; - /* Ineligible saved state falls back to the vote account. */ - ctx->promotion.source = FD_FAILOVER_SOURCE_VOTE_ACCOUNT; + if( FD_UNLIKELY( ctx->promotion.from_peer || !ctx->promotion.force ) ) return 0; + if( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) { + FD_LOG_NOTICE(( "%s: %s is not eligible (%s), trying %s", ctx->promotion.label, + source_name( ctx->promotion.source ), reason, source_name( FD_FAILOVER_SOURCE_EMPTY ) )); + if( ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ctx->peer_history.valid = 0; + /* Forced recovery falls back to a bound covering known votes. */ + ctx->promotion.source = FD_FAILOVER_SOURCE_EMPTY; } else { - FD_LOG_WARNING(( "%s: the vote account cannot be adopted (%s), --force proceeds with an empty tower", ctx->promotion.label, reason )); - /* Only a forced operator promotion may fall back to empty history. */ - ctx->promotion.empty = 1; + return 0; } ctx->promotion.adopt.valid = 0; ctx->promotion.adopt.tip = FD_FAILOVER_SLOT_NULL; ctx->promotion.adopt.sz = 0UL; - ctx->current_tower = ctx->promotion.adopt; + ctx->current_history = ctx->promotion.adopt; ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; ctx->promotion.retry = 0; ctx->adopt_result_fresh = 0; + promote_start( ctx ); /* Retry adoption from the fallback source after the prior attempt has ended. */ ctx->action = FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY; return 1; } -/* Returns whether the adopted tower reaches the coverage floor, the - highest vote the peer reported or we signed. The vote account also - passes once replay is past the floor by the slack. */ +/* Returns whether adoption covers the highest vote the peer reported or + we signed. Forced empty history covers it through its vote bound. */ static int floor_covered( fd_failover_tile_ctx_t const * ctx ) { ulong floor = ctx->promotion.floor; - ulong tip = ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT ? ctx->adopt_result.acct_vote_slot - : ctx->adopt_result.vote_slot; - if( FD_LIKELY( floor==FD_FAILOVER_SLOT_NULL ) ) return 1; - if( FD_LIKELY( tip!=FD_FAILOVER_SLOT_NULL && tip>=floor ) ) return 1; - return ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT && - ctx->replay_slot!=FD_FAILOVER_SLOT_NULL && - ctx->replay_slot>fd_ulong_sat_add( floor, FD_FAILOVER_FLOOR_SLACK_SLOTS ); + ulong tip = ctx->promotion.source==FD_FAILOVER_SOURCE_EMPTY ? ctx->adopt_result.vote_bound + : ctx->adopt_result.vote_slot; + return floor==FD_FAILOVER_SLOT_NULL || ( tip!=FD_FAILOVER_SLOT_NULL && tip>=floor ); } /* Whether an active peer authenticated after this promotion started, @@ -1302,11 +1319,11 @@ static void step_demote_drain( fd_failover_tile_ctx_t * ctx ) { deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); int expired = deadline_expired( ctx ) || ( ctx->handoff.until && fd_failover_clock()>=ctx->handoff.until ); - /* The watermark is the tower tile's output sequence when it stopped + /* The watermark is the votor's output sequence when it stopped signing. */ - if( FD_UNLIKELY( !fd_seq_ge( fd_seq_inc( ctx->tower_seen_seq, 1UL ), ctx->id_switch.result.tower_watermark ) ) ) { + if( FD_UNLIKELY( !fd_seq_ge( fd_seq_inc( ctx->hist_seen_seq, 1UL ), ctx->id_switch.result.history_watermark ) ) ) { if( FD_UNLIKELY( expired ) ) { - FD_LOG_WARNING(( "handoff %lu: the tower did not report its last votes in time, up to seq %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->id_switch.result.tower_watermark )); + FD_LOG_WARNING(( "handoff %lu: the vote history stream did not report its last votes in time, up to seq %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->id_switch.result.history_watermark )); ctx->id_switch.fresh = 0; ctx->id_switch.overdue = 0; set_role( ctx, FD_FAILOVER_ROLE_STANDBY ); @@ -1353,7 +1370,7 @@ step_demote_switch( fd_failover_tile_ctx_t * ctx, return; } /* The junk key is installed everywhere. */ - FD_LOG_NOTICE(( "handoff %lu: the junk identity is installed, waiting for the tower to drain up to seq %lu", ctx->handoff.id, ctx->id_switch.result.tower_watermark )); + FD_LOG_NOTICE(( "handoff %lu: the junk identity is installed, waiting for the vote history stream to drain up to seq %lu", ctx->handoff.id, ctx->id_switch.result.history_watermark )); ctx->action = FD_FAILOVER_ACTION_DEMOTE_DRAIN; demotion_deadline_start( ctx ); step_demote_drain( ctx ); @@ -1371,7 +1388,7 @@ step_demote_wait_ack( fd_failover_tile_ctx_t * ctx ) { ctx->session.peer_boot_id!=ctx->handoff.target.boot_id ) ) { FD_LOG_WARNING(( "the peer restarted before it answered handoff %lu and may have voted before it restarted, neither machine holds the staked identity now, " "nothing votes until `failover promote --force` runs on one of them", ctx->handoff.id )); - ctx->peer_tower.valid = 0; + ctx->peer_history.valid = 0; /* A new peer boot ends this ACK wait without proving whether it voted. */ handoff_resolved( ctx, FD_FAILOVER_HANDOFF_RESTARTED ); return; @@ -1400,11 +1417,10 @@ step_promote_wait_replay( fd_failover_tile_ctx_t * ctx, reject_promotion( ctx, FD_FAILOVER_REJECT_REPLAY_BEHIND, 0 ); return; } - /* Wait for replay to reach the tower's tip before adopting, otherwise - the tower refers to blocks we have not seen yet. */ + /* Replay must start and reach the history's finality anchor. */ if( FD_UNLIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) ) return; - if( FD_UNLIKELY( ctx->promotion.adopt.tip!=FD_FAILOVER_SLOT_NULL && ctx->replay_slotpromotion.adopt.tip ) ) { - (void)promote_fallback( ctx, "replay has not reached its tip" ); + if( FD_UNLIKELY( !replay_ready( ctx ) ) ) { + if( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) (void)promote_fallback( ctx, "replay has not reached its finality anchor" ); return; } ulong id = publish_adopt_state( ctx, stem ); @@ -1413,8 +1429,7 @@ step_promote_wait_replay( fd_failover_tile_ctx_t * ctx, reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } - FD_LOG_NOTICE(( "%s: asking the tower tile to adopt %s", ctx->promotion.label, - ctx->promotion.empty ? "an empty tower authorized by --force" : source_name( ctx->promotion.source ) )); + FD_LOG_NOTICE(( "%s: asking the votor to adopt %s", ctx->promotion.label, source_name( ctx->promotion.source ) )); ctx->promotion.adopt_id = id; ctx->promotion.retry = 0; /* The adoption request is published, wait for its response. */ @@ -1433,8 +1448,7 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, return; } if( FD_UNLIKELY( ctx->promotion.retry ) ) { - /* The tower tile has not replayed every block the tower votes on, or - the adopted tip is short of the floor. */ + /* Wait for replay before retrying a recoverable adoption error. */ if( FD_LIKELY( ctx->replay_slot<=ctx->promotion.retry_slot ) ) return; ulong id = publish_adopt_state( ctx, stem ); if( FD_UNLIKELY( id==ULONG_MAX ) ) { @@ -1450,16 +1464,17 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, if( FD_LIKELY( !ctx->adopt_result_fresh ) ) return; /* Wait for an adoption response. */ ctx->adopt_result_fresh = 0; if( FD_UNLIKELY( ctx->adopt_result_id!=ctx->promotion.adopt_id ) ) return; /* Ignore responses to earlier attempts. */ - if( FD_UNLIKELY( ctx->adopt_result.result==FD_TOWER_ADOPT_ERR_UNREPLAYED ) ) { + if( FD_UNLIKELY( ctx->adopt_result.result==FD_VOTOR_ADOPT_ERR_UNREPLAYED || + ctx->adopt_result.result==FD_VOTOR_ADOPT_ERR_CAPACITY ) ) { /* Operator promotion tries a fallback before waiting for more replay. */ - if( promote_fallback( ctx, "its blocks have not been replayed" ) ) return; + if( promote_fallback( ctx, adopt_err_name( ctx->adopt_result.result ) ) ) return; if( FD_UNLIKELY( !ctx->promotion.from_peer ) ) { /* End the failed adoption attempt while remaining standby. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } if( FD_UNLIKELY( !ctx->promotion.retry_logged ) ) { - FD_LOG_NOTICE(( "%s: the tower tile has not replayed every block the tower votes on, asking again as replay moves", ctx->promotion.label )); + FD_LOG_NOTICE(( "%s: the votor is not ready to adopt the vote history (%s), asking again as replay moves", ctx->promotion.label, adopt_err_name( ctx->adopt_result.result ) )); ctx->promotion.retry_logged = 1; } /* Keep the bound final state and wait for replay before retrying. */ @@ -1467,40 +1482,31 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, ctx->promotion.retry_slot = ctx->replay_slot; return; } - if( FD_UNLIKELY( ctx->adopt_result.result!=FD_TOWER_ADOPT_SUCCESS ) ) { + if( FD_UNLIKELY( ctx->adopt_result.result!=FD_VOTOR_ADOPT_SUCCESS ) ) { /* Try the next permitted source before failing promotion. */ if( promote_fallback( ctx, adopt_err_name( ctx->adopt_result.result ) ) ) return; - FD_LOG_WARNING(( "%s: the tower tile refused %s (%s)", ctx->promotion.label, source_name( ctx->promotion.source ), adopt_err_name( ctx->adopt_result.result ) )); + FD_LOG_WARNING(( "%s: the votor refused %s (%s)", ctx->promotion.label, source_name( ctx->promotion.source ), adopt_err_name( ctx->adopt_result.result ) )); ulong result = ctx->adopt_result.result; - /* Tower rejected the history, stop promotion and report any handoff refusal. */ - reject_promotion( ctx, result==FD_TOWER_ADOPT_ERR_BLOCK_MISMATCH ? FD_FAILOVER_REJECT_ADOPTION_MISMATCH - : FD_FAILOVER_REJECT_ADOPTION_FAILED, - result==FD_TOWER_ADOPT_ERR_BLOCK_MISMATCH || result==FD_TOWER_ADOPT_ERR_DECODE || - result==FD_TOWER_ADOPT_ERR_INVALID ); + /* Votor rejected the history, stop promotion and report any handoff refusal. */ + reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, + result==FD_VOTOR_ADOPT_ERR_DECODE || result==FD_VOTOR_ADOPT_ERR_INVALID || result==FD_VOTOR_ADOPT_ERR_STALE ); return; } if( FD_UNLIKELY( ctx->promotion.adopt.tip!=FD_FAILOVER_SLOT_NULL && ctx->adopt_result.vote_slot!=ctx->promotion.adopt.tip ) ) { /* Operator promotion may retry a different source after a tip mismatch. */ if( promote_fallback( ctx, "adoption did not retain its final vote" ) ) return; - FD_LOG_WARNING(( "%s: the adopted tower ends at slot %lu, the tower says %lu, our root has passed its last vote, so it is a mismatch", ctx->promotion.label, ctx->adopt_result.vote_slot, ctx->promotion.adopt.tip )); + FD_LOG_WARNING(( "%s: the adopted vote history ends at slot %lu, the history says %lu, so it is a mismatch", ctx->promotion.label, ctx->adopt_result.vote_slot, ctx->promotion.adopt.tip )); /* The adopted tip differs, fail promotion without installing the staked key. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_MISMATCH, 1 ); return; } - int covered = floor_covered( ctx ); - int empty_account = ctx->promotion.source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT && ctx->adopt_result.acct_vote_slot==FD_FAILOVER_SLOT_NULL; - if( FD_UNLIKELY( !ctx->promotion.from_peer && ( !covered || empty_account ) ) ) { - /* Insufficient saved history falls back to the vote account. */ - if( ctx->promotion.source!=FD_FAILOVER_SOURCE_VOTE_ACCOUNT && - promote_fallback( ctx, "its votes do not cover the known signing history" ) ) return; - if( !ctx->promotion.empty ) { - char tip[ 32 ], floor[ 32 ]; - FD_LOG_WARNING(( "%s: --force accepts %s vote-account history (last account vote %s, required coverage floor %s), earlier votes and lockouts may be lost", ctx->promotion.label, - empty_account ? "empty" : "incomplete", slot_text( ctx->adopt_result.acct_vote_slot, tip ), slot_text( ctx->promotion.floor, floor ) )); - } + log_adopted( ctx ); + int covered = floor_covered( ctx ); + if( FD_UNLIKELY( !covered && !ctx->promotion.from_peer && ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) { + if( promote_fallback( ctx, "its votes do not cover the known signing history" ) ) return; } if( FD_UNLIKELY( !covered && !ctx->promotion.force ) ) { - /* Only a handoff gets here. The tip of the peer's tower does not + /* Only a handoff gets here. The tip of the peer's history does not change, so we do not ask again and the deadline ends that wait. */ if( FD_UNLIKELY( !ctx->promotion.floor_logged ) ) { FD_LOG_WARNING(( "%s: %s ends at slot %lu, short of the coverage floor at slot %lu, the promotion stops at its deadline", ctx->promotion.label, @@ -1527,8 +1533,7 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } - FD_LOG_NOTICE(( "%s: the tower tile adopted %s, asking the admin tile to install the staked identity", ctx->promotion.label, - ctx->promotion.empty ? "an empty tower authorized by --force" : source_name( ctx->promotion.source ) )); + FD_LOG_NOTICE(( "%s: asking the admin tile to install the staked identity", ctx->promotion.label )); /* Adoption passed, wait for admin to install the staked identity. */ ctx->action = FD_FAILOVER_ACTION_PROMOTE_SWITCH; return; @@ -1561,17 +1566,16 @@ step_promote_switch( fd_failover_tile_ctx_t * ctx ) { } /* Admin confirmed the staked identity is installed, become active. */ set_role( ctx, FD_FAILOVER_ROLE_ACTIVE ); - /* A new tenure starts, the peer tower we kept is older than its votes. */ - ctx->peer_tower.valid = 0; - ctx->taken = 0; + /* A new tenure starts, the peer history we kept is older than its votes. */ + ctx->peer_history.valid = 0; + ctx->taken = 0; /* Local promotion is complete. */ ctx->action = FD_FAILOVER_ACTION_IDLE; ctx->stuck = 0; - if( FD_UNLIKELY( ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ) ) { - FD_LOG_NOTICE(( "%s: we are the active now, the staked identity is installed with %s", ctx->promotion.label, - ctx->promotion.empty ? "an empty tower authorized by --force" : "the vote account's tower" )); + if( FD_LIKELY( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) { + FD_LOG_NOTICE(( "%s: we are the active now, the staked identity is installed and the adopted vote history ends at slot %lu", ctx->promotion.label, ctx->last_vote_slot )); } else { - FD_LOG_NOTICE(( "%s: we are the active now, the staked identity is installed and the adopted tower ends at slot %lu", ctx->promotion.label, ctx->last_vote_slot )); + FD_LOG_NOTICE(( "%s: we are the active now, the staked identity is installed with an empty history, no vote at or below slot %lu", ctx->promotion.label, ctx->empty_vote_after )); } if( FD_UNLIKELY( ctx->promotion.from_peer ) ) { /* The ACK tells the old active its handoff is done. We keep it as @@ -1734,29 +1738,30 @@ handle_control( fd_failover_tile_ctx_t * ctx, ulong result = FD_ADMINCTL_RESULT_SUCCESS; if( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT || ctx->tx.valid || ctx->session.close_after_send ) result = FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; else if( ctx->role!=FD_FAILOVER_ROLE_ACTIVE || peer->role!=FD_FAILOVER_ROLE_STANDBY ) result = FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; - else if( !final_tower_ok( ctx ) ) result = FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER; - /* A standby this far behind could not reach our last vote before its - promotion deadline. */ + else if( !final_history_ok( ctx ) ) result = FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY; + /* A standby this far behind could not reach the slot its promotion + waits for before the deadline. */ else if( request.replay_slot==FD_FAILOVER_SLOT_NULL || - fd_ulong_sat_add( request.replay_slot, FD_FAILOVER_DEADLINE_SLOTS )last_vote_slot ) result = FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND; + fd_ulong_sat_add( request.replay_slot, FD_FAILOVER_DEADLINE_SLOTS )=ctx->handoff_refuse_log_at ) { + if( result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY && now>=ctx->handoff_refuse_log_at ) { ctx->handoff_refuse_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); char tip[ 32 ], last[ 32 ]; - char const * why = ctx->tower_gap ? "a vote-history update was skipped, the final state cannot be verified" : + char const * why = ctx->hist_gap ? "a vote-history update was skipped, the final state cannot be verified" : ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ? "no vote is known to the failover controller, it cannot determine whether this machine has not voted or is unable to vote" : - !ctx->current_tower.valid || !ctx->current_tower.sz ? "a vote is known but its final state is unavailable" : + !ctx->current_history.valid || !ctx->current_history.sz ? "a vote is known but its final state is unavailable" : "the saved final state does not match the latest known vote"; FD_LOG_WARNING(( "handoff %lu refused: %s (latest known vote %s, saved history tip %s), this machine keeps the staked identity, check local voting progress and vote-history logs before retrying", request.handoff_id, why, slot_text( ctx->last_vote_slot, last ), - slot_text( ctx->current_tower.valid ? ctx->current_tower.tip : FD_FAILOVER_SLOT_NULL, tip ) )); + slot_text( ctx->current_history.valid ? ctx->current_history.tip : FD_FAILOVER_SLOT_NULL, tip ) )); } else if( result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND && now>=ctx->handoff_refuse_log_at ) { ctx->handoff_refuse_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); char replay[ 32 ]; - FD_LOG_WARNING(( "handoff %lu refused: the standby's replay is at slot %s, more than %lu slots behind our last vote at slot %lu, this machine keeps the staked identity, " + FD_LOG_WARNING(( "handoff %lu refused: the standby's replay is at slot %s, more than %lu slots behind %s at slot %lu, this machine keeps the staked identity, " "run `failover promote` there again once it has caught up", - request.handoff_id, slot_text( request.replay_slot, replay ), FD_FAILOVER_DEADLINE_SLOTS, ctx->last_vote_slot )); + request.handoff_id, slot_text( request.replay_slot, replay ), FD_FAILOVER_DEADLINE_SLOTS, + "our finality anchor", ready_slot( ctx ) )); } handoff_result( ctx, request.handoff_id, result ); return; @@ -1802,9 +1807,12 @@ handle_control( fd_failover_tile_ctx_t * ctx, } case (ushort)FD_FAILOVER_MSG_DEMOTED: { - /* The peer says it can no longer sign and hands us its tower. */ + /* The peer says it can no longer sign and hands us its history. */ fd_failover_demoted_t demoted; if( FD_UNLIKELY( !fd_failover_demoted_decode( &demoted, ctx->rx, payload_sz ) ) ) break; + /* HELLO pairs only members in the same mode, so a DEMOTED in the + other mode is malformed. */ + if( FD_UNLIKELY( demoted.mode!=(uchar)FD_FAILOVER_MODE_ALPENGLOW ) ) break; ulong boot_id = ctx->session.peer_boot_id; fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); /* Bind before deduplication. An unsolicited DEMOTED cannot change @@ -1819,8 +1827,8 @@ handle_control( fd_failover_tile_ctx_t * ctx, queue_reply( ctx ); return; } - fd_failover_tower_t tower = { .valid=1, .tip=demoted.last_vote_slot, .sz=(ulong)demoted.state_len }; - fd_memcpy( tower.state, ctx->rx+sizeof(fd_failover_demoted_t), tower.sz ); + fd_failover_history_t history = { .valid=1, .tip=demoted.last_vote_slot, .sz=(ulong)demoted.state_len }; + fd_memcpy( history.state, ctx->rx+sizeof(fd_failover_demoted_t), history.sz ); uchar reason = FD_FAILOVER_REJECT_NONE; char const * why = NULL; if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_STANDBY ) ) { @@ -1836,9 +1844,9 @@ handle_control( fd_failover_tile_ctx_t * ctx, finish_reply( ctx, &ctx->request.peer, demoted.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason ); return; } - FD_LOG_NOTICE(( "handoff %lu: accepted the peer's final tower ending at slot %lu, waiting for local adoption before taking the identity", demoted.handoff_id, tower.tip )); - if( ctx->peer_floor==FD_FAILOVER_SLOT_NULL || tower.tip>ctx->peer_floor ) ctx->peer_floor = tower.tip; - ctx->peer_tower = tower; + FD_LOG_NOTICE(( "handoff %lu: accepted the peer's final vote history ending at slot %lu, waiting for local adoption before taking the identity", demoted.handoff_id, history.tip )); + if( ctx->peer_floor==FD_FAILOVER_SLOT_NULL || history.tip>ctx->peer_floor ) ctx->peer_floor = history.tip; + ctx->peer_history = history; ctx->session.peer_role = FD_FAILOVER_ROLE_STANDBY; /* The bound final state moves the standby from peer wait to replay wait. */ start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &ctx->request.peer, demoted.handoff_id, 0 ); @@ -1858,15 +1866,15 @@ handle_control( fd_failover_tile_ctx_t * ctx, ctx->taken = 1; ctx->session.peer_role = FD_FAILOVER_ROLE_ACTIVE; ctx->stuck = 0; - /* The peer votes past the final tower we handed it from here on. */ - ctx->peer_tower.valid = 0; + /* The peer votes past the final history we handed it from here on. */ + ctx->peer_history.valid = 0; handoff_result( ctx, ack.handoff_id, FD_ADMINCTL_RESULT_SUCCESS ); return; } case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: { /* The peer did not take the identity. Either machine may now recover - without telling the other, so we drop any tower the peer handed us + without telling the other, so we drop any history the peer handed us earlier. */ fd_failover_promote_rejected_t rej; if( FD_UNLIKELY( !fd_failover_promote_rejected_decode( &rej, ctx->rx, payload_sz ) ) ) break; @@ -1878,7 +1886,7 @@ handle_control( fd_failover_tile_ctx_t * ctx, FD_LOG_WARNING(( "the peer declined handoff %lu (%s), neither machine holds the staked identity now, nothing votes until `failover promote --force` runs on one of them, " "see the peer's log first", rej.handoff_id, reject_name( rej.reason ) )); } - ctx->peer_tower.valid = 0; + ctx->peer_history.valid = 0; /* The peer refusal ends the demotion wait, this machine stays standby. */ handoff_resolved( ctx, FD_FAILOVER_HANDOFF_DECLINED ); ctx->stuck = 1; @@ -2056,16 +2064,16 @@ promote_guard( fd_failover_tile_ctx_t const * ctx, return FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED; } -/* The tower source a promotion would adopt now. */ +/* The history source a promotion would adopt now. */ static ulong promote_source( fd_failover_tile_ctx_t const * ctx ) { /* The peer may already vote past our unanswered DEMOTED. */ - if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) return FD_FAILOVER_SOURCE_VOTE_ACCOUNT; + if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) return FD_FAILOVER_SOURCE_EMPTY; ulong floor = coverage_floor( ctx ); ulong root = ctx->root_slot; - int peer_ok = ctx->peer_tower.valid && ( floor==FD_FAILOVER_SLOT_NULL || ctx->peer_tower.tip>=floor ) && - ( root ==FD_FAILOVER_SLOT_NULL || ctx->peer_tower.tip> root ); - return peer_ok ? FD_FAILOVER_SOURCE_PEER : FD_FAILOVER_SOURCE_VOTE_ACCOUNT; + int peer_ok = ctx->peer_history.valid && ( floor==FD_FAILOVER_SLOT_NULL || ctx->peer_history.tip>=floor ) && + ( root ==FD_FAILOVER_SLOT_NULL || ctx->peer_history.tip> root ); + return peer_ok ? FD_FAILOVER_SOURCE_PEER : FD_FAILOVER_SOURCE_EMPTY; } /* Whether our handoff request is paused at its deadline. */ @@ -2161,24 +2169,26 @@ apply_control( fd_failover_tile_ctx_t * ctx, } if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) { FD_LOG_WARNING(( "`failover promote --force` stops waiting for the peer's answer to handoff %lu, discarding saved final state because the peer may have voted, coverage floors retained", ctx->handoff.id )); - ctx->peer_tower.valid = 0; + ctx->peer_history.valid = 0; /* Forced recovery ends the ACK wait before starting local promotion. */ handoff_resolved( ctx, FD_FAILOVER_HANDOFF_CANCELLED ); } if( force ) FD_LOG_WARNING(( "operator promotion: --force asserts the peer cannot sign, peer checks are bypassed and incomplete or empty vote history is permitted if needed" )); - if( source==FD_FAILOVER_SOURCE_VOTE_ACCOUNT ) { - fd_failover_tower_t const * saved = &ctx->peer_tower; + if( source==FD_FAILOVER_SOURCE_EMPTY ) { + fd_failover_history_t const * saved = &ctx->peer_history; + char const * stored = "the stored peer vote history"; if( saved->valid ) { ulong floor = coverage_floor( ctx ); if( floor!=FD_FAILOVER_SLOT_NULL && saved->tiptip, floor )); + FD_LOG_NOTICE(( "operator promotion: %s ends at slot %lu below the known signing floor %lu, skipping it", stored, saved->tip, floor )); else if( ctx->root_slot!=FD_FAILOVER_SLOT_NULL && saved->tip<=ctx->root_slot ) - FD_LOG_NOTICE(( "operator promotion: the stored peer tower ends at slot %lu at or below the local root %lu, skipping it", saved->tip, ctx->root_slot )); + FD_LOG_NOTICE(( "operator promotion: %s ends at slot %lu at or below the local root %lu, skipping it", stored, saved->tip, ctx->root_slot )); } - FD_LOG_NOTICE(( "operator promotion: no eligible saved final tower is available, selecting the vote account" )); + FD_LOG_NOTICE(( "operator promotion: no eligible saved final state is available, selecting an empty history" )); } else { - fd_failover_tower_t const * saved = &ctx->peer_tower; - FD_LOG_NOTICE(( "operator promotion: selecting %s through slot %lu, the tower tile must validate and adopt it before the staked identity is installed", source_name( source ), saved->tip )); + fd_failover_history_t const * saved = &ctx->peer_history; + FD_LOG_NOTICE(( "operator promotion: selecting %s through slot %lu, the %s must validate and adopt it before the staked identity is installed", + source_name( source ), saved->tip, "votor" )); } /* The accepted operator command starts the replay and adoption sequence. */ start_promotion( ctx, source, NULL, 0UL, force ); @@ -2277,10 +2287,10 @@ gossip_commit( fd_failover_tile_ctx_t * ctx, if( FD_UNLIKELY( addr==ctx->own_gossip.addr && ctx->gossip_socket.port==ctx->own_gossip.port ) ) return; ctx->staked_seen_at = now; /* Gossip may be the only sign of a peer tenure, so we drop the peer's - tower for good, not only while the contact info is fresh. */ - if( FD_UNLIKELY( ctx->peer_tower.valid ) ) + history for good, not only while the contact info is fresh. */ + if( FD_UNLIKELY( ctx->peer_history.valid ) ) FD_LOG_NOTICE(( "gossip shows the staked identity at another host, discarding saved final state from before its voting tenure, keeping coverage floors" )); - ctx->peer_tower.valid = 0; + ctx->peer_history.valid = 0; if( FD_LIKELY( addr==ctx->staked_addr ) ) return; uint old_addr = ctx->staked_addr; @@ -2298,10 +2308,10 @@ gossip_commit( fd_failover_tile_ctx_t * ctx, static void input_overrun( fd_failover_tile_ctx_t * ctx ) { if( FD_UNLIKELY( ctx->input_lost ) ) return; - ctx->input_lost = 1; - ctx->stuck = 1; - ctx->slot_done_fresh = 0; - ctx->bus_req_fresh = 0; + ctx->input_lost = 1; + ctx->stuck = 1; + ctx->hist_fresh = 0; + ctx->bus_req_fresh = 0; fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); fd_failover_channel_hangup( ctx->channel, fd_failover_clock() ); FD_LOG_WARNING(( "failover input overrun, failover is disabled until restart, validator processing continues" )); @@ -2316,15 +2326,10 @@ before_frag( fd_failover_tile_ctx_t * ctx, if( FD_LIKELY( in_idx==ctx->gossip_in_idx ) ) { return sig!=FD_GOSSIP_UPDATE_TAG_CONTACT_INFO && sig!=FD_GOSSIP_UPDATE_TAG_CONTACT_INFO_REMOVE; } - if( FD_LIKELY( in_idx==ctx->tower_in_idx ) ) { - /* A skipped frag may have been a vote. A slot done counts toward the - watermark in after_frag, once its vote is in the cache. Other - frags hold no vote and count here. */ - if( FD_UNLIKELY( ctx->tower_seen_seq!=ULONG_MAX && seq!=fd_seq_inc( ctx->tower_seen_seq, 1UL ) ) ) ctx->tower_gap = 1; - if( FD_UNLIKELY( sig!=FD_TOWER_SIG_SLOT_DONE ) ) { - ctx->tower_seen_seq = seq; - return 1; - } + if( FD_LIKELY( in_idx==ctx->hist_in_idx ) ) { + /* Every history frame counts toward the drain watermark after the + stem validates it. A skipped frame may have contained a vote. */ + if( FD_UNLIKELY( ctx->hist_seen_seq!=ULONG_MAX && seq!=fd_seq_inc( ctx->hist_seen_seq, 1UL ) ) ) ctx->hist_gap = 1; return 0; } if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { @@ -2343,11 +2348,12 @@ during_frag( fd_failover_tile_ctx_t * ctx, ulong chunk, ulong sz, ulong ctl FD_PARAM_UNUSED ) { + ctx->read_pending = 1; if( FD_UNLIKELY( in_idx==ctx->adopt_in_idx ) ) { - if( FD_UNLIKELY( chunkadopt_in_chunk0 || chunk>ctx->adopt_in_wmark || sz!=sizeof(fd_tower_adopt_result_t) ) ) { + if( FD_UNLIKELY( chunkadopt_in_chunk0 || chunk>ctx->adopt_in_wmark || sz!=sizeof(fd_votor_adopt_result_t) ) ) { FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->adopt_in_chunk0, ctx->adopt_in_wmark )); } - fd_memcpy( &ctx->adopt_result, fd_chunk_to_laddr_const( ctx->adopt_in_mem, chunk ), sizeof(fd_tower_adopt_result_t) ); + fd_memcpy( &ctx->adopt_result, fd_chunk_to_laddr_const( ctx->adopt_in_mem, chunk ), sizeof(fd_votor_adopt_result_t) ); return; } if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { @@ -2367,11 +2373,12 @@ during_frag( fd_failover_tile_ctx_t * ctx, fd_memcpy( &ctx->bus_req, msg, sizeof(fd_failover_bus_msg_t) ); return; } - if( FD_UNLIKELY( in_idx==ctx->tower_in_idx ) ) { - if( FD_UNLIKELY( chunktower_in_chunk0 || chunk>ctx->tower_in_wmark || sz!=sizeof(fd_tower_msg_t) ) ) { - FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->tower_in_chunk0, ctx->tower_in_wmark )); + if( FD_UNLIKELY( in_idx==ctx->hist_in_idx ) ) { + ulong frame_sz = sizeof(fd_votor_hist_msg_t); + if( FD_UNLIKELY( chunkhist_in_chunk0 || chunk>ctx->hist_in_wmark || sz!=frame_sz ) ) { + FD_LOG_ERR(( "chunk %lu %lu corrupt, not in range [%lu,%lu]", chunk, sz, ctx->hist_in_chunk0, ctx->hist_in_wmark )); } - fd_memcpy( &ctx->slot_done, fd_chunk_to_laddr_const( ctx->tower_in_mem, chunk ), sizeof(fd_tower_slot_done_t) ); + fd_memcpy( &ctx->hist, fd_chunk_to_laddr_const( ctx->hist_in_mem, chunk ), sizeof(fd_votor_hist_msg_t) ); return; } if( FD_UNLIKELY( in_idx!=ctx->gossip_in_idx ) ) return; @@ -2391,6 +2398,12 @@ during_frag( fd_failover_tile_ctx_t * ctx, } } +/* Unknown adoption results are invalid. */ +static ulong +votor_adopt_code( ulong result ) { + return resultread_pending = 0; if( FD_UNLIKELY( in_idx==ctx->adopt_in_idx ) ) { - ctx->adopt_result_id = sig; - ctx->adopt_result_fresh = 1; + ctx->adopt_result.result = votor_adopt_code( ctx->adopt_result.result ); + ctx->adopt_result_id = sig; + ctx->adopt_result_fresh = 1; return; } if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { @@ -2421,9 +2436,9 @@ after_frag( fd_failover_tile_ctx_t * ctx, ctx->bus_req_fresh = 1; return; } - if( FD_UNLIKELY( in_idx==ctx->tower_in_idx ) ) { - ctx->slot_done_fresh = 1; - ctx->tower_seen_seq = seq; + if( FD_UNLIKELY( in_idx==ctx->hist_in_idx ) ) { + ctx->hist_fresh = 1; + ctx->hist_seen_seq = seq; return; } if( FD_UNLIKELY( in_idx!=ctx->gossip_in_idx ) ) return; @@ -2435,6 +2450,8 @@ after_credit( fd_failover_tile_ctx_t * ctx, fd_stem_context_t * stem, int * opt_poll_in FD_PARAM_UNUSED, int * charge_busy ) { + /* STEM skips after_frag if the fragment was overwritten during the read. */ + if( FD_UNLIKELY( ctx->read_pending ) ) input_overrun( ctx ); if( FD_UNLIKELY( ctx->input_lost ) ) { if( FD_UNLIKELY( ctx->bus_req_fresh ) ) { ctx->bus_req_fresh = 0; @@ -2443,9 +2460,9 @@ after_credit( fd_failover_tile_ctx_t * ctx, } return; } - if( FD_UNLIKELY( ctx->slot_done_fresh ) ) { - ctx->slot_done_fresh = 0; - consume_slot_done( ctx, &ctx->slot_done ); + if( FD_UNLIKELY( ctx->hist_fresh ) ) { + ctx->hist_fresh = 0; + consume_hist( ctx, &ctx->hist ); } step_controller( ctx, stem ); @@ -2504,26 +2521,6 @@ populate_allowed_fds( fd_topo_t const * topo, return out_cnt; } -static void -metrics_write( fd_failover_tile_ctx_t * ctx ) { - fd_failover_channel_metrics_t const * m = fd_failover_channel_metrics( ctx->channel ); - FD_MCNT_SET( FAILOV, CONN_ACCEPTED, m->accept_cnt ); - FD_MCNT_SET( FAILOV, ACCEPT_BUDGET_EXHAUSTED, m->accept_limited_cnt ); - FD_MCNT_SET( FAILOV, CONN_REJECTED_SOURCE, m->source_reject_cnt ); - FD_MCNT_SET( FAILOV, CONN_REJECTED_BUDGET, m->budget_reject_cnt ); - FD_MCNT_SET( FAILOV, CONN_REJECTED_CAPACITY, m->capacity_reject_cnt ); - FD_MCNT_SET( FAILOV, CONN_RECONNECT_ADMITTED, m->reconnect_cnt ); - FD_MCNT_SET( FAILOV, HANDSHAKE_TIMED_OUT, m->handshake_timeout_cnt ); - FD_MCNT_SET( FAILOV, PROOF_REJECTED, m->proof_reject_cnt ); - FD_MCNT_SET( FAILOV, HELLO_REJECTED, m->hello_reject_cnt ); - FD_MCNT_SET( FAILOV, TLS_HANDSHAKE_FAILED, m->tls_fail_cnt ); - FD_MCNT_SET( FAILOV, FRAME_PROTOCOL_REJECTED, m->wire_fatal_cnt ); - FD_MCNT_SET( FAILOV, CONN_AUTHENTICATED, m->paired_cnt ); - FD_MCNT_SET( FAILOV, HANDSHAKE_DURATION_NANOS, m->handshake_duration_nanos ); - FD_MGAUGE_SET( FAILOV, HANDSHAKE_LATEST_DURATION_NANOS, m->handshake_latest_nanos ); - FD_MGAUGE_SET( FAILOV, CONN_PENDING, fd_failover_channel_pending( ctx->channel ) ); -} - /* An iteration publishes at most an adopt or switch request and a bus response, within the burst of 3. */ #define STEM_BURST (3UL) @@ -2535,9 +2532,7 @@ metrics_write( fd_failover_tile_ctx_t * ctx ) { #define STEM_CALLBACK_CONTEXT_TYPE fd_failover_tile_ctx_t #define STEM_CALLBACK_CONTEXT_ALIGN alignof(fd_failover_tile_ctx_t) -#define STEM_CALLBACK_METRICS_WRITE metrics_write #define STEM_CALLBACK_AFTER_POLL_OVERRUN input_overrun -#define STEM_CALLBACK_AFTER_READ_OVERRUN input_overrun #define STEM_CALLBACK_AFTER_CREDIT after_credit #define STEM_CALLBACK_BEFORE_FRAG before_frag #define STEM_CALLBACK_DURING_FRAG during_frag diff --git a/src/discof/failover/test_failover_channel.c b/src/discof/failover/test_failover_channel.c index 38a73ffcc47..4985d1dc7c6 100644 --- a/src/discof/failover/test_failover_channel.c +++ b/src/discof/failover/test_failover_channel.c @@ -28,7 +28,7 @@ member( ulong idx, ulong staked_idx ) { FD_TEST( ch ); fd_failover_hello_t hello = { .version=FD_FAILOVER_VERSION, .boot_id=idx+1UL, .role=(uchar)(idx ? FD_FAILOVER_ROLE_STANDBY : FD_FAILOVER_ROLE_ACTIVE), - .mode=FD_FAILOVER_MODE_TOWER }; + .mode=FD_FAILOVER_MODE_ALPENGLOW }; fd_memcpy( hello.junk_pubkey, keys[idx]+32UL, 32UL ); fd_memcpy( hello.staked_pubkey, keys[staked_idx]+32UL, 32UL ); fd_memset( hello.vote_account, 0xBB, 32UL ); diff --git a/src/discof/failover/test_failover_proto.c b/src/discof/failover/test_failover_proto.c index 7ea44cf91fa..b72cdfa97d8 100644 --- a/src/discof/failover/test_failover_proto.c +++ b/src/discof/failover/test_failover_proto.c @@ -1,4 +1,5 @@ #include "fd_failover_proto.h" +#include "../../choreo/votor/ag_hist.h" #include "../../util/fd_util.h" static void @@ -13,7 +14,7 @@ fill_hello( fd_failover_hello_t * h, fd_memset( h->staked_pubkey, staked, 32UL ); fd_memset( h->vote_account, vote, 32UL ); h->role = role; - h->mode = (uchar)FD_FAILOVER_MODE_TOWER; + h->mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; h->boot_id = 0x1000UL+junk; } @@ -24,6 +25,37 @@ test_hello( void ) { fill_hello( &standby, 2U, 0xAA, 0xBB, FD_FAILOVER_ROLE_STANDBY ); FD_TEST( fd_failover_hello_check( &active, &standby )==FD_FAILOVER_HELLO_OK ); FD_TEST( fd_failover_hello_check( &standby, &active )==FD_FAILOVER_HELLO_OK ); + uchar unsupported[] = { 0U, 2U, 255U }; + for( ulong i=0UL; imode = 0U; + FD_TEST( !fd_failover_demoted_decode( &decoded, payload, sz ) ); + header->mode = FD_FAILOVER_MODE_ALPENGLOW; + header->last_vote_slot = 102UL; + FD_TEST( !fd_failover_demoted_decode( &decoded, payload, sz ) ); } static void @@ -56,6 +88,7 @@ int main( int argc, char ** argv ) { fd_boot( &argc, &argv ); test_hello(); + test_demoted_decode(); test_wrong_staked_key(); test_handoff_decode(); FD_LOG_NOTICE(( "pass" )); diff --git a/src/discof/failover/test_failover_tile.c b/src/discof/failover/test_failover_tile.c index 0367f2c431c..1d31e46e4bb 100644 --- a/src/discof/failover/test_failover_tile.c +++ b/src/discof/failover/test_failover_tile.c @@ -44,6 +44,7 @@ controller_init( ulong role ) { fd_memset( ctx, 0, sizeof(fd_failover_tile_ctx_t) ); ctx->role = role; ctx->hello.role = (uchar)role; + ctx->hello.mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; ctx->hello.boot_id = OUR_BOOT_ID; fd_memset( ctx->hello.junk_pubkey, 0x11, 32UL ); fd_memset( ctx->hello.staked_pubkey, 0x5A, 32UL ); @@ -54,10 +55,12 @@ controller_init( ulong role ) { ctx->deadline_slot = FD_FAILOVER_SLOT_NULL; ctx->peer_floor = FD_FAILOVER_SLOT_NULL; ctx->own_floor = FD_FAILOVER_SLOT_NULL; - ctx->tower_seen_seq = ULONG_MAX; + ctx->adopt_anchor = FD_FAILOVER_SLOT_NULL; + ctx->empty_vote_after = FD_FAILOVER_SLOT_NULL; + ctx->hist_seen_seq = ULONG_MAX; ctx->handoff_base = 5000UL; ctx->gossip_in_idx = ULONG_MAX; - ctx->tower_in_idx = ULONG_MAX; + ctx->hist_in_idx = ULONG_MAX; ctx->admin_in_idx = ULONG_MAX; ctx->adopt_in_idx = ULONG_MAX; ctx->admin_out_idx = 0UL; @@ -92,17 +95,14 @@ pair( fd_failover_tile_ctx_t * ctx, } static void -make_tower( fd_failover_tower_t * tower, +make_history( fd_failover_history_t * history, ulong tip ) { - fd_compact_tower_sync_serde_t serde; - fd_memset( &serde, 0, sizeof(serde) ); - serde.root = tip-2UL; - serde.lockouts_cnt = 1; - serde.lockouts[ 0 ].offset = 2UL; - serde.lockouts[ 0 ].confirmation_count = 1; - FD_TEST( !fd_compact_tower_sync_ser( &serde, tower->state, FD_FAILOVER_TOWER_STATE_MAX, &tower->sz ) ); - tower->valid = 1; - tower->tip = tip; + ag_hist_t hist = { .anchor=tip, .last_leader_slot=FD_FAILOVER_SLOT_NULL, .vote_bound=FD_FAILOVER_SLOT_NULL, .rec_cnt=1UL }; + hist.rec[ 0 ].slot = tip; + hist.rec[ 0 ].flags = AG_HIST_FLAG_VOTED; + FD_TEST( !ag_hist_ser( &hist, history->state, sizeof(history->state), &history->sz ) ); + history->valid = 1; + history->tip = tip; } static void @@ -124,8 +124,8 @@ deliver_ack( fd_failover_tile_ctx_t * ctx, static void switch_ok( fd_failover_tile_ctx_t * ctx, ulong watermark ) { - ctx->id_switch.result.result = FD_FAILOVER_SWITCH_OK; - ctx->id_switch.result.tower_watermark = watermark; + ctx->id_switch.result.result = FD_FAILOVER_SWITCH_OK; + ctx->id_switch.result.history_watermark = watermark; FD_TEST( switch_answer( ctx, ctx->id_switch.request_id ) ); } @@ -141,7 +141,7 @@ static void test_handoff( void ) { fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); - make_tower( &ctx->current_tower, 99UL ); + make_history( &ctx->current_history, 99UL ); fd_failover_handoff_request_t req = { .handoff_id=42UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH ); @@ -168,7 +168,7 @@ test_no_final_state( void ) { fd_failover_handoff_result_t result; FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); - FD_TEST( result.handoff_id==43UL && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_TOWER ); + FD_TEST( result.handoff_id==43UL && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY ); FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE ); FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.request_id && !ctx->promotion.adopt_id ); controller_fini( ctx ); @@ -182,7 +182,7 @@ test_request_replay_behind( void ) { fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); ctx->last_vote_slot = 200UL; - make_tower( &ctx->current_tower, 200UL ); + make_history( &ctx->current_history, 200UL ); ulong behind[ 2 ] = { 200UL-FD_FAILOVER_DEADLINE_SLOTS-1UL, FD_FAILOVER_SLOT_NULL }; for( ulong i=0UL; i<2UL; i++ ) { fd_failover_handoff_request_t req = { .handoff_id=49UL+i, .target_boot_id=OUR_BOOT_ID, .replay_slot=behind[ i ] }; @@ -208,11 +208,11 @@ static void test_demote_drain( void ) { fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); - make_tower( &ctx->current_tower, 99UL ); + make_history( &ctx->current_history, 99UL ); fd_failover_handoff_request_t req = { .handoff_id=44UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); step_controller( ctx, stem ); - ctx->tower_seen_seq = 796UL; + ctx->hist_seen_seq = 796UL; switch_ok( ctx, 800UL ); step_controller( ctx, stem ); FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN ); @@ -222,7 +222,7 @@ test_demote_drain( void ) { deliver_ack( ctx, 44UL ); deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN && !ctx->taken && !ctx->tx.valid ); - ctx->tower_seen_seq = 799UL; + ctx->hist_seen_seq = 799UL; step_controller( ctx, stem ); FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && pending_demoted( ctx ).handoff_id==44UL ); controller_fini( ctx ); @@ -259,7 +259,7 @@ static void test_operator_switch( void ) { fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); - make_tower( &ctx->current_tower, 99UL ); + make_history( &ctx->current_history, 99UL ); ctx->own_floor = 99UL; fd_failover_handoff_request_t req = { .handoff_id=47UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); @@ -272,7 +272,7 @@ test_operator_switch( void ) { FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && ctx->hello.role==FD_FAILOVER_ROLE_STANDBY ); FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->stuck && !ctx->tx.valid && !ctx->handoff.id && !ctx->request.id ); FD_TEST( ctx->hello.boot_id!=OUR_BOOT_ID && ctx->channel->self_hello.boot_id==ctx->hello.boot_id ); - FD_TEST( !ctx->current_tower.valid && ctx->own_floor==99UL ); + FD_TEST( !ctx->current_history.valid && ctx->own_floor==99UL ); FD_TEST( ctx->id_switch.epoch==1UL && ctx->id_switch.discard && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK ); FD_TEST( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ); @@ -305,14 +305,14 @@ test_operator_switch( void ) { FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.fresh && ctx->action==FD_FAILOVER_ACTION_IDLE ); /* A request bound to our first boot is refused, one to this boot is - served once we have a final tower. */ + served once we have a final history. */ pair( ctx, 78UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE ); ctx->tx.valid = 0; ctx->session.close_after_send = 0; ctx->last_vote_slot = 120UL; - make_tower( &ctx->current_tower, 120UL ); + make_history( &ctx->current_history, 120UL ); fd_failover_handoff_request_t fresh = { .handoff_id=48UL, .target_boot_id=ctx->hello.boot_id, .replay_slot=120UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &fresh, sizeof(fresh) ); FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->handoff.id==48UL ); @@ -354,19 +354,20 @@ test_operator_failover_identity( void ) { controller_fini( ctx ); } -/* test_standby_vote_counts: a vote our tower published counts toward +/* test_standby_vote_counts: a vote our history published counts toward the floor whatever our role, set-identity can install the staked key before we hear of it. */ static void test_standby_vote_counts( void ) { fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); - fd_tower_slot_done_t done; - fd_memset( &done, 0, sizeof(done) ); - done.replay_slot = 130UL; - done.root_slot = FD_FAILOVER_SLOT_NULL; - done.has_vote_txn = 1; - done.vote_slot = 129UL; - consume_slot_done( ctx, &done ); + fd_votor_hist_msg_t hist; + fd_memset( &hist, 0, sizeof(hist) ); + fd_memcpy( hist.identity.uc, ctx->hello.staked_pubkey, 32UL ); + hist.replay_slot = 130UL; + hist.root_slot = FD_FAILOVER_SLOT_NULL; + hist.has_vote = 1; + hist.vote_slot = 129UL; + consume_hist( ctx, &hist ); FD_TEST( ctx->own_floor==129UL && ctx->last_vote_slot==129UL ); controller_fini( ctx ); } @@ -433,7 +434,7 @@ test_session_change_drops_result( void ) { ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); - make_tower( &ctx->current_tower, 99UL ); + make_history( &ctx->current_history, 99UL ); req.target_boot_id = OUR_BOOT_ID; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); step_controller( ctx, stem ); @@ -476,7 +477,7 @@ test_request_address( void ) { FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,4) && dial_port( ctx )==9001 ); request_pause( ctx, 1000L, "test" ); status_snapshot( ctx, 1000L, &status ); - FD_TEST( status.request_paused && status.mode==(uchar)FD_FAILOVER_MODE_TOWER ); + FD_TEST( status.request_paused && status.mode==(uchar)FD_FAILOVER_MODE_ALPENGLOW ); FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, FD_IP4_ADDR(10,0,0,5), 0 )==FD_ADMINCTL_RESULT_SUCCESS ); FD_TEST( ctx->request.addr==FD_IP4_ADDR(10,0,0,5) && dial_port( ctx )==9001 ); status_snapshot( ctx, 1000L, &status ); @@ -506,7 +507,7 @@ static void test_abort_answers_requester( void ) { fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); - make_tower( &ctx->current_tower, 99UL ); + make_history( &ctx->current_history, 99UL ); fd_failover_handoff_request_t req = { .handoff_id=47UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); step_controller( ctx, stem ); diff --git a/src/discof/failover/test_failover_tile_ag.c b/src/discof/failover/test_failover_tile_ag.c new file mode 100644 index 00000000000..8c1797f6973 --- /dev/null +++ b/src/discof/failover/test_failover_tile_ag.c @@ -0,0 +1,302 @@ +#include "fd_failover_channel.c" +#include "fd_failover_tile.c" + +#define OWN_GOSSIP_ADDR FD_IP4_ADDR(10,0,0,1) +#define OWN_GOSSIP_PORT ((ushort)8001) + +static fd_frag_meta_t pub_mcache[ 8 ]; +static ulong pub_cr_avail; +static ulong pub_min_cr_avail; +static int pub_reliable; +static fd_stem_context_t stem[ 1 ]; +static uchar bus_mem[ 8192 ] __attribute__((aligned(128))); + +static void +stem_init( void ) { + static fd_frag_meta_t * mcaches[ 1 ]; + static ulong seqs[ 1 ]; + static ulong depths[ 1 ]; + mcaches[ 0 ] = pub_mcache; + seqs[ 0 ] = 0UL; + depths[ 0 ] = 8UL; + pub_cr_avail = 64UL; + pub_min_cr_avail = 64UL; + pub_reliable = 0; + *stem = (fd_stem_context_t){ + .mcaches = mcaches, .seqs = seqs, .depths = depths, + .cr_avail = &pub_cr_avail, .min_cr_avail = &pub_min_cr_avail, + .cr_decrement_amount = 1UL, .out_reliable = &pub_reliable, + }; +} + +static uchar ch_mem[ 32UL<<20 ] __attribute__((aligned(128))); +static fd_failover_tile_ctx_t ctl[ 1 ]; + +#define OUR_BOOT_ID (1000UL) + +static fd_failover_tile_ctx_t * +controller_init( ulong role ) { + stem_init(); + fd_failover_tile_ctx_t * ctx = ctl; + fd_memset( ctx, 0, sizeof(fd_failover_tile_ctx_t) ); + ctx->role = role; + ctx->hello.role = (uchar)role; + ctx->hello.mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; + ctx->hello.boot_id = OUR_BOOT_ID; + fd_memset( ctx->hello.junk_pubkey, 0x11, 32UL ); + fd_memset( ctx->hello.staked_pubkey, 0x5A, 32UL ); + ctx->replay_slot = 100UL; + ctx->root_slot = 90UL; + ctx->last_vote_slot = 99UL; + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_CNT; + ctx->deadline_slot = FD_FAILOVER_SLOT_NULL; + ctx->peer_floor = FD_FAILOVER_SLOT_NULL; + ctx->own_floor = FD_FAILOVER_SLOT_NULL; + ctx->adopt_anchor = FD_FAILOVER_SLOT_NULL; + ctx->empty_vote_after = FD_FAILOVER_SLOT_NULL; + ctx->hist_seen_seq = ULONG_MAX; + ctx->handoff_base = 5000UL; + ctx->gossip_in_idx = ULONG_MAX; + ctx->hist_in_idx = ULONG_MAX; + ctx->admin_in_idx = ULONG_MAX; + ctx->adopt_in_idx = ULONG_MAX; + ctx->admin_out_idx = 0UL; + ctx->admin_out_mem = (fd_wksp_t *)bus_mem; + ctx->adopt_out_idx = 0UL; + ctx->adopt_out_mem = (fd_wksp_t *)bus_mem; + ctx->own_gossip.addr = OWN_GOSSIP_ADDR; + ctx->own_gossip.port = fd_ushort_bswap( OWN_GOSSIP_PORT ); + ctx->channel = fd_failover_channel_join( fd_failover_channel_new( ch_mem ) ); + FD_TEST( ctx->channel ); + ctx->channel->self_hello = ctx->hello; + ctx->session.state = fd_failover_channel_state( ctx->channel ); + return ctx; +} + +static void +controller_fini( fd_failover_tile_ctx_t * ctx ) { + fd_failover_channel_fini( ctx->channel ); +} + +static void +pair( fd_failover_tile_ctx_t * ctx, + ulong boot_id, + ulong peer_role, + long now ) { + ctx->channel->state = FD_FAILOVER_SESSION_PAIRED; + ctx->channel->peer_hello.boot_id = boot_id; + ctx->channel->peer_hello.role = (uchar)peer_role; + sync_session( ctx ); + (void)now; + fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x22, 32UL ); +} + +/* A vote history of rec_cnt voted slots ending at tip. */ +static void +make_hist( ag_hist_t * hist, + ulong anchor, + ulong last_leader_slot, + ulong tip, + ulong rec_cnt ) { + FD_TEST( rec_cnt && rec_cnt<=AG_HIST_MAX ); + fd_memset( hist, 0, sizeof(*hist) ); + hist->anchor = anchor; + hist->last_leader_slot = last_leader_slot; + hist->rec_cnt = rec_cnt; + for( ulong i=0UL; irec[ i ].slot = tip-rec_cnt+1UL+i; + hist->rec[ i ].flags = AG_HIST_FLAG_VOTED|AG_HIST_FLAG_VOTED_NOTAR; + fd_memset( hist->rec[ i ].notar_hash, (int)(0x10UL+i), sizeof(ag_block_hash_t) ); + } +} + +/* The same history serialized into history, as the active keeps it. */ +static void +make_history( fd_failover_history_t * history, + ulong anchor, + ulong tip, + ulong rec_cnt ) { + static ag_hist_t hist; + make_hist( &hist, anchor, tip-3UL, tip, rec_cnt ); + FD_TEST( !ag_hist_ser( &hist, history->state, FD_FAILOVER_ALPENGLOW_STATE_MAX, &history->sz ) ); + history->valid = 1; + history->tip = tip; +} + +static void +deliver( fd_failover_tile_ctx_t * ctx, + ushort type, + void const * payload, + ulong payload_sz ) { + fd_memcpy( ctx->rx, payload, payload_sz ); + handle_control( ctx, type, payload_sz, 1000L ); +} + +static void +deliver_ack( fd_failover_tile_ctx_t * ctx, + ulong handoff_id ) { + uchar payload[ sizeof(fd_failover_promote_ack_t) ]; + deliver( ctx, (ushort)FD_FAILOVER_MSG_PROMOTE_ACK, payload, fd_failover_promote_ack_encode( payload, handoff_id ) ); +} + +static void +switch_ok( fd_failover_tile_ctx_t * ctx, + ulong watermark ) { + ctx->id_switch.result.result = FD_FAILOVER_SWITCH_OK; + ctx->id_switch.result.history_watermark = watermark; + FD_TEST( switch_answer( ctx, ctx->id_switch.request_id ) ); +} + +static fd_failover_demoted_t +pending_demoted( fd_failover_tile_ctx_t const * ctx ) { + FD_TEST( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ); + fd_failover_demoted_t demoted; + FD_TEST( fd_failover_demoted_decode( &demoted, ctx->tx.payload, ctx->tx.sz ) ); + return demoted; +} + +/* test_handoff: the active hands over its vote history in an alpenglow + DEMOTED. */ +static void +test_handoff( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + make_history( &ctx->current_history, 90UL, 99UL, 6UL ); + fd_failover_handoff_request_t req = { .handoff_id=42UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH ); + step_controller( ctx, stem ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK && !ctx->tx.valid ); + switch_ok( ctx, 0UL ); + step_controller( ctx, stem ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY ); + fd_failover_demoted_t demoted = pending_demoted( ctx ); + FD_TEST( demoted.handoff_id==42UL && demoted.mode==FD_FAILOVER_MODE_ALPENGLOW && demoted.last_vote_slot==99UL ); + ctx->tx.valid = 0; + ctx->handoff.delivery = FD_FAILOVER_DELIVERY_SENT; + deliver_ack( ctx, 42UL ); + FD_TEST( ctx->taken && ctx->handoff.result==FD_FAILOVER_HANDOFF_TAKEN ); + controller_fini( ctx ); +} + +/* test_no_final_history: an active with no vote declines the handoff. */ +static void +test_no_final_history( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_handoff_request_t req = { .handoff_id=43UL, .target_boot_id=OUR_BOOT_ID }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + fd_failover_handoff_result_t result; + FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); + FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); + FD_TEST( result.handoff_id==43UL && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.request_id && !ctx->promotion.adopt_id ); + char const * hint; + FD_TEST( !strcmp( control_refusal( result.result, &hint ), "NO_FINAL_HISTORY" ) ); + FD_TEST( strstr( hint, "the active" ) ); + /* The requester logs the relayed rejection under the same name. */ + FD_TEST( !strcmp( relayed_refusal( result.result, &hint ), "NO_FINAL_HISTORY" ) ); + controller_fini( ctx ); +} + +/* test_request_replay_behind: under alpenglow a request is served while + replay is within the slot deadline of the history's finality anchor. */ +static void +test_request_replay_behind( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + ctx->last_vote_slot = 300UL; + make_history( &ctx->current_history, 200UL, 300UL, 6UL ); + fd_failover_handoff_request_t req = { .handoff_id=52UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=200UL-FD_FAILOVER_DEADLINE_SLOTS-1UL }; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + fd_failover_handoff_result_t result; + FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); + FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); + FD_TEST( result.handoff_id==52UL && result.result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->handoff.id ); + ctx->tx.valid = 0; + ctx->session.close_after_send = 0; + req.handoff_id = 53UL; + req.replay_slot = 200UL-FD_FAILOVER_DEADLINE_SLOTS; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->handoff.id==53UL ); + controller_fini( ctx ); +} + +/* test_force_empty_bound: a forced promotion with no saved history sets + its vote bound at the replay slot or the coverage floor, whichever is + higher. */ +static void +test_force_empty_bound( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_cstr_ncpy( ctx->promotion.label, "promotion 1", sizeof(ctx->promotion.label) ); + ctx->promotion.source = FD_FAILOVER_SOURCE_EMPTY; + ctx->promotion.force = 1; + ulong const floors[ 3 ] = { FD_FAILOVER_SLOT_NULL, 120UL, 80UL }; + ulong const fences[ 3 ] = { 100UL, 120UL, 100UL }; + for( ulong i=0UL; i<3UL; i++ ) { + ctx->promotion.floor = floors[ i ]; + promote_start( ctx ); + FD_TEST( replay_ready( ctx ) ); + FD_TEST( ctx->empty_vote_after==fences[ i ] ); + FD_TEST( ctx->promotion.adopt.sz==FD_VOTOR_ADOPT_EMPTY_SZ && FD_LOAD( ulong, ctx->promotion.adopt.state )==fences[ i ] ); + } + controller_fini( ctx ); +} + +static void +test_adopt_capacity_retry( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + make_history( &ctx->peer_history, 90UL, 99UL, 6UL ); + fd_failover_peer_t peer = { .boot_id = 77UL }; + start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &peer, 42UL, 0 ); + step_promote_wait_replay( ctx, stem ); + ulong id = ctx->promotion.adopt_id; + fd_failover_history_t saved = ctx->promotion.adopt; + ctx->adopt_in_idx = 0UL; + ctx->adopt_result.result = FD_VOTOR_ADOPT_ERR_CAPACITY; + after_frag( ctx, 0UL, 0UL, id, sizeof(ctx->adopt_result), 0UL, 0UL, stem ); + step_promote_wait_adopt( ctx, stem ); + FD_TEST( ctx->promotion.retry && ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ); + FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && !ctx->stuck && !ctx->tx.valid ); + FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ); + FD_TEST( ctx->peer_history.valid && fd_memeq( &saved, &ctx->promotion.adopt, sizeof(saved) ) ); + step_promote_wait_adopt( ctx, stem ); + FD_TEST( ctx->promotion.adopt_id==id ); + + ctx->replay_slot++; + step_promote_wait_adopt( ctx, stem ); + FD_TEST( !ctx->promotion.retry && ctx->promotion.adopt_id!=id ); + FD_TEST( fd_memeq( bus_mem, saved.state, saved.sz ) ); + ctx->adopt_result = (fd_votor_adopt_result_t){ .result=FD_VOTOR_ADOPT_SUCCESS, .root=90UL, .vote_slot=99UL, .vote_bound=ULONG_MAX }; + after_frag( ctx, 0UL, 1UL, ctx->promotion.adopt_id, sizeof(ctx->adopt_result), 0UL, 0UL, stem ); + step_promote_wait_adopt( ctx, stem ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH ); + controller_fini( ctx ); +} + +/* Unknown votor results are refused, and stale history is named explicitly. */ +static void +test_votor_codes( void ) { + for( ulong r=FD_VOTOR_ADOPT_SUCCESS; rvalidator_cnt; rank++ ) { if( FD_UNLIKELY( !memcmp( epoch_info->validators[ rank ].id_key, ctx->id_key.uc, sizeof(ag_id_key_t) ) ) ) own_rank = (ushort)rank; } + if( FD_UNLIKELY( ctx->failover_enabled && !ctx->vote_authority ) ) own_rank = USHORT_MAX; ag_pool_set_rank ( ctx->pool, epoch_slots[ i ], own_rank ); ag_votor_set_rank( ctx->votor, epoch_slots[ i ], own_rank ); } @@ -1159,6 +1191,7 @@ update_own_bls_key( fd_votor_tile_t * ctx ) { for( ulong rank=0UL; rankvalidator_cnt; rank++ ) { if( FD_UNLIKELY( !memcmp( epoch_info->validators[ rank ].id_key, ctx->id_key.uc, sizeof(ag_id_key_t) ) ) ) own_rank = (ushort)rank; } + if( FD_UNLIKELY( ctx->failover_enabled && !ctx->vote_authority ) ) own_rank = USHORT_MAX; /* under failover we stay unranked while we may not vote */ uchar const * own_bls_key = NULL; /* NULL if we are not ranked or cannot sign with our key, in which case we do not vote */ if( FD_LIKELY( own_rank!=USHORT_MAX ) ) { ag_validator_info_t const * validator = &epoch_info->validators[ own_rank ]; @@ -1329,6 +1362,277 @@ rank_voters( ag_epoch_info_t * epoch_info, return epoch_info; } +/* Returns the highest slot whose window this identity may have led, + ULONG_MAX when there is none. */ +static ulong +leader_floor( fd_votor_tile_t const * ctx ) { + if( ctx->last_leader_slot==ULONG_MAX ) return ctx->adopted_last_leader_slot; + if( ctx->adopted_last_leader_slot==ULONG_MAX ) return ctx->last_leader_slot; + return fd_ulong_max( ctx->last_leader_slot, ctx->adopted_last_leader_slot ); +} + +/* Publishes a votor_hist frame to the failover tile, see + fd_votor_hist_msg. */ +static void +publish_hist( fd_votor_tile_t * ctx, + fd_stem_context_t * stem, + int has_vote ) { + if( FD_UNLIKELY( ctx->hist_out_idx==ULONG_MAX ) ) return; + fd_votor_hist_msg_t * msg = fd_chunk_to_laddr( ctx->hist_out_mem, ctx->hist_out_chunk ); + msg->identity = ctx->id_key; + msg->replay_slot = ctx->highest_completed_slot ? ctx->highest_completed_slot : ULONG_MAX; + msg->root_slot = ctx->root_slot; + msg->has_vote = has_vote; + /* Preserve the identity switch floor for the next adopter. */ + if( FD_UNLIKELY( ctx->vote_history_slot ) ) ag_votor_set_vote_bound( ctx->votor, ctx->vote_history_slot-1UL ); + msg->truncated = ag_votor_hist_export( ctx->votor, leader_floor( ctx ), &msg->hist ); + msg->vote_slot = ag_hist_tip( &msg->hist ); /* the failover tile checks the history ends here */ + if( FD_UNLIKELY( msg->truncated && !ctx->hist_was_truncated ) ) FD_LOG_WARNING(( "vote history export dropped windows below slot %lu, finality is far behind our votes", ag_hist_first_slot( msg->hist.anchor ) )); + ctx->hist_was_truncated = msg->truncated; + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->hist_out_idx, FD_VOTOR_HIST_SIG, ctx->hist_out_chunk, sizeof(fd_votor_hist_msg_t), 0UL, tspub, tspub ); + ctx->hist_out_chunk = fd_dcache_compact_next( ctx->hist_out_chunk, sizeof(fd_votor_hist_msg_t), ctx->hist_out_chunk0, ctx->hist_out_wmark ); + ctx->hist_seq = stem->seqs[ ctx->hist_out_idx ]; +} + +/* Raises the adopted leader floor to the window of slot, so we do not + lead that window or one below it. */ +static void +raise_leader_floor( fd_votor_tile_t * ctx, + ulong slot ) { + if( FD_UNLIKELY( slot==ULONG_MAX ) ) return; + ulong window = ag_first_slot_in_window( slot ); + if( FD_LIKELY( ctx->adopted_last_leader_slot==ULONG_MAX || ctx->adopted_last_leader_slotadopted_last_leader_slot = window; +} + +/* An empty request is shorter than any serialized history. */ +FD_STATIC_ASSERT( FD_VOTOR_ADOPT_EMPTY_SZsizeof(ctx->adopt_req) || ag_hist_de( req, req_sz, hist ) ) ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history of %lu bytes that does not decode", req_sz )); + return result; + } + + result.result = FD_VOTOR_ADOPT_ERR_UNREPLAYED; + if( FD_UNLIKELY( !ctx->init ) ) { + FD_LOG_WARNING(( "failover: not adopting the vote history, consensus is not up yet" )); + return result; + } + result.result = FD_VOTOR_ADOPT_ERR_INVALID; + + /* An empty request sets the vote bound and leader floor to the slot it + gives. */ + if( FD_UNLIKELY( empty ) ) { + ulong bound = FD_LOAD( ulong, req ); + if( FD_UNLIKELY( bound==ULONG_MAX ) ) { + FD_LOG_WARNING(( "failover: refusing an empty vote history that has no bound slot" )); + return result; + } + ag_votor_set_vote_bound( ctx->votor, bound ); + raise_leader_floor( ctx, bound ); + ctx->failover_hist_adopted = 1; + result.result = FD_VOTOR_ADOPT_SUCCESS; + result.root = ag_votor_metrics( ctx->votor ).highest_final_cert_slot; + result.vote_bound = ag_votor_vote_bound( ctx->votor ); + FD_LOG_NOTICE(( "failover: starting from an empty vote history on anchor %lu, no vote at or below slot %lu", result.root, result.vote_bound )); + return result; + } + + ulong tip = ag_hist_tip( hist ); + if( FD_UNLIKELY( tip==ULONG_MAX ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history on anchor %lu that has no votes in it", hist->anchor )); + return result; + } + + /* Reject a history older than the votes we sent as the staked identity. */ + result.result = FD_VOTOR_ADOPT_ERR_STALE; + if( FD_UNLIKELY( !ctx->normal_startup && ctx->last_vote_slot!=ULONG_MAX && tiplast_vote_slot ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history through slot %lu, this machine already voted as this identity through slot %lu", tip, ctx->last_vote_slot )); + return result; + } + + /* Reject a finality anchor past the blocks we replayed. */ + result.result = FD_VOTOR_ADOPT_ERR_UNREPLAYED_ROOT; + if( FD_UNLIKELY( ctx->highest_completed_slot && hist->anchor>ctx->highest_completed_slot ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history on anchor %lu, replay has only reached slot %lu, retry the promotion once replay passes the anchor", hist->anchor, ctx->highest_completed_slot )); + return result; + } + + ulong conflicts = ag_votor_hist_adopt( ctx->votor, hist ); + if( FD_UNLIKELY( conflicts==ULONG_MAX ) ) { + result.result = FD_VOTOR_ADOPT_ERR_CAPACITY; + FD_LOG_WARNING(( "failover: vote history exceeds the available slot state capacity, retry as consensus prunes old slots" )); + return result; + } + if( FD_UNLIKELY( conflicts ) ) FD_LOG_WARNING(( "adopted vote history disagrees with %lu of our own notar hashes, theirs were the ones sent", conflicts )); + ctx->failover_hist_adopted = 1; + raise_leader_floor( ctx, hist->last_leader_slot ); + raise_leader_floor( ctx, ag_votor_vote_bound( ctx->votor ) ); + result.result = FD_VOTOR_ADOPT_SUCCESS; + result.root = ag_votor_metrics( ctx->votor ).highest_final_cert_slot; + result.vote_slot = tip; + result.vote_bound = ag_votor_vote_bound( ctx->votor ); + if( FD_LIKELY( result.vote_bound==ULONG_MAX ) ) FD_LOG_NOTICE(( "failover: adopted a vote history through slot %lu, the anchor is now %lu", tip, result.root )); + else FD_LOG_NOTICE(( "failover: adopted a vote history through slot %lu, the anchor is now %lu, no vote at or below slot %lu", tip, result.root, result.vote_bound )); + return result; +} + +/* Applies the failover voting rule at the end of an identity switch. */ +static void +failover_switched( fd_votor_tile_t * ctx ) { + /* Skip leader windows this identity may already have led. */ + ulong floor = leader_floor( ctx ); + if( FD_UNLIKELY( floor!=ULONG_MAX ) ) { + ulong next_window = ag_first_slot_in_window( ctx->highest_parent_ready_slot )+AG_SLOTS_PER_WINDOW; + ulong floor_window = ag_first_slot_in_window( floor )+AG_SLOTS_PER_WINDOW; + if( FD_UNLIKELY( floor_window>next_window ) ) { + ctx->next_leader_slot = fd_multi_epoch_leaders_get_next_slot( ctx->mleaders, fd_ulong_max( fd_ulong_max( ctx->curr_epoch_slot, floor_window ), ctx->vote_history_slot ), &ctx->id_key ); + } + } + FD_BASE58_ENCODE_32_BYTES( ctx->id_key.uc, id_key_b58 ); + int operator = !!FD_VOLATILE_CONST( ctx->id_keyswitch->operator ); + int standby = fd_pubkey_eq( &ctx->id_key, &ctx->boot_id_key ); + ctx->vote_authority = ( ctx->normal_startup && operator ) || ( !standby && ( operator || ctx->failover_hist_adopted ) ); + FD_LOG_NOTICE(( "failover: switched to identity `%s`%s, this machine is now %s", id_key_b58, operator ? " by set-identity" : "", + ctx->vote_authority ? "the active voter" : "a standby" )); + if( FD_UNLIKELY( !operator && !standby && !ctx->failover_hist_adopted ) ) FD_LOG_WARNING(( "failover: the staked identity was installed without an adopted vote history, refusing to vote until an adoption precedes an identity switch" )); + ctx->failover_hist_adopted = 0; + ctx->id_keyswitch->result = ctx->hist_seq; +} + +/* Records the slot replay completed and its root. */ +static void +failover_slot_completed( fd_votor_tile_t * ctx, + fd_replay_slot_completed_t const * slot_completed ) { + ctx->highest_completed_slot = fd_ulong_max( ctx->highest_completed_slot, slot_completed->slot ); + if( FD_LIKELY( slot_completed->root_slot!=ULONG_MAX ) ) ctx->root_slot = slot_completed->root_slot; +} + +/* Records a vote we cast and publishes the history, marking the vote + unsent if it never left the machine. */ +static void +failover_after_vote( fd_votor_tile_t * ctx, + fd_stem_context_t * stem, + ulong vote_slot, + int sent ) { + if( FD_LIKELY( sent && ( ctx->last_vote_slot==ULONG_MAX || vote_slot>ctx->last_vote_slot ) ) ) ctx->last_vote_slot = vote_slot; + if( FD_UNLIKELY( !sent ) ) ag_votor_mark_unsent( ctx->votor, &ctx->scratch.vote ); + publish_hist( ctx, stem, sent ); +} + +/* Records our LEADER and publishes the history. */ +static void +failover_after_leader( fd_votor_tile_t * ctx, + fd_stem_context_t * stem ) { + ctx->last_leader_slot = ctx->next_leader_slot; + publish_hist( ctx, stem, 0 ); +} + +/* failover_init initializes failover in the votor tile, all of it off + unless the topology has the failover links. */ +static void +failover_init( fd_votor_tile_t * ctx, + fd_topo_t const * topo, + fd_topo_tile_t const * tile ) { + /* Without the failover links failover stays off. */ + ulong failov_in_idx = fd_topo_find_tile_in_link ( topo, tile, "failov_votor", tile->kind_id ); + ctx->hist_out_idx = fd_topo_find_tile_out_link( topo, tile, "votor_hist", tile->kind_id ); + ctx->failov_out_idx = fd_topo_find_tile_out_link( topo, tile, "votor_failov", tile->kind_id ); + ctx->failover_enabled = failov_in_idx!=ULONG_MAX; + if( FD_UNLIKELY( ( ctx->hist_out_idx!=ULONG_MAX )!=ctx->failover_enabled || ( ctx->failov_out_idx!=ULONG_MAX )!=ctx->failover_enabled ) ) { + FD_LOG_ERR(( "votor tile needs all of the failov_votor, votor_hist and votor_failov links or none of them" )); + } + if( FD_UNLIKELY( ctx->failover_enabled ) ) { + fd_topo_link_t const * hist_out = &topo->links[ tile->out_link_id[ ctx->hist_out_idx ] ]; + ctx->hist_out_mem = topo->workspaces[ topo->objs[ hist_out->dcache_obj_id ].wksp_id ].wksp; + ctx->hist_out_chunk0 = fd_dcache_compact_chunk0( ctx->hist_out_mem, hist_out->dcache ); + ctx->hist_out_wmark = fd_dcache_compact_wmark ( ctx->hist_out_mem, hist_out->dcache, hist_out->mtu ); + ctx->hist_out_chunk = ctx->hist_out_chunk0; + fd_topo_link_t const * failov_out = &topo->links[ tile->out_link_id[ ctx->failov_out_idx ] ]; + ctx->failov_out_mem = topo->workspaces[ topo->objs[ failov_out->dcache_obj_id ].wksp_id ].wksp; + ctx->failov_out_chunk0 = fd_dcache_compact_chunk0( ctx->failov_out_mem, failov_out->dcache ); + ctx->failov_out_wmark = fd_dcache_compact_wmark ( ctx->failov_out_mem, failov_out->dcache, failov_out->mtu ); + ctx->failov_out_chunk = ctx->failov_out_chunk0; + } + + /* A configured standby waits for a switch. An outgoing client keeps + ordinary voting until its first successful adoption. */ + ctx->boot_id_key = ctx->id_key; + ctx->normal_startup = !tile->votor.failover_standby; + ctx->vote_authority = !tile->votor.failover_standby; + ctx->failover_hist_adopted = 0; + ctx->hist_was_truncated = 0; + ctx->last_leader_slot = ULONG_MAX; + ctx->adopted_last_leader_slot = ULONG_MAX; + ctx->last_vote_slot = ULONG_MAX; + ctx->root_slot = ULONG_MAX; + ctx->adopt_req_sz = 0UL; + ctx->hist_seq = 0UL; + ctx->highest_completed_slot = 0UL; +} + +/* Copies an adoption request from the failover tile, after_frag responds + to it. */ +static void +failover_adopt_copy( fd_votor_tile_t * ctx, + ulong in_idx, + ulong chunk, + ulong sz ) { + ctx->adopt_req_sz = sz; + if( FD_LIKELY( sz<=sizeof(ctx->adopt_req) ) ) fd_memcpy( ctx->adopt_req, fd_chunk_to_laddr_const( ctx->in[ in_idx ].mem, chunk ), sz ); +} + +/* Responds to an adoption request, echoing its sequence number. */ +static void +failover_adopt_answer( fd_votor_tile_t * ctx, + ulong sig, + fd_stem_context_t * stem ) { + fd_votor_adopt_result_t result; + if( FD_UNLIKELY( !fd_pubkey_eq( &ctx->id_key, &ctx->boot_id_key ) || ( ctx->vote_authority && !ctx->normal_startup ) ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history adoption outside standby or client startup" )); + result = (fd_votor_adopt_result_t){ .result=FD_VOTOR_ADOPT_ERR_INVALID, .root=ULONG_MAX, .vote_slot=ULONG_MAX, .vote_bound=ULONG_MAX }; + } else { + /* The client's old notar votes belong to its boot identity. Fence + their whole windows so the target cannot finalize them. Export + the bound with future histories so another machine preserves it. */ + ulong client_bound = ULONG_MAX; + if( FD_UNLIKELY( ctx->normal_startup ) ) { + ag_hist_t local; + ag_votor_hist_export( ctx->votor, ULONG_MAX, &local ); + ulong tip = ag_hist_tip( &local ); + if( tip!=ULONG_MAX ) client_bound = ag_first_slot_in_window( tip )+AG_SLOTS_PER_WINDOW-1UL; + } + result = failover_adopt_hist( ctx, ctx->adopt_req, ctx->adopt_req_sz ); + if( FD_LIKELY( result.result==FD_VOTOR_ADOPT_SUCCESS ) ) { + if( FD_UNLIKELY( ctx->normal_startup ) ) { + if( client_bound!=ULONG_MAX ) { + ag_votor_set_vote_bound( ctx->votor, client_bound ); + raise_leader_floor( ctx, client_bound ); + result.vote_bound = ag_votor_vote_bound( ctx->votor ); + } + ctx->last_vote_slot = ULONG_MAX; + } + ctx->normal_startup = 0; + ctx->vote_authority = 0; + update_own_rank( ctx ); + update_own_bls_key( ctx ); + } + } + fd_memcpy( fd_chunk_to_laddr( ctx->failov_out_mem, ctx->failov_out_chunk ), &result, sizeof(result) ); + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->failov_out_idx, sig, ctx->failov_out_chunk, sizeof(result), 0UL, tspub, tspub ); + ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(result), ctx->failov_out_chunk0, ctx->failov_out_wmark ); +} + static void handle_epoch( fd_votor_tile_t * ctx, fd_epoch_info_msg_t const * msg ) { @@ -1485,6 +1789,7 @@ handle_epoch( fd_votor_tile_t * ctx, ctx->ns_per_slot = (long)msg->ns_per_slot; ctx->slots_per_epoch = msg->epoch_schedule.slots_per_epoch; + if( FD_UNLIKELY( ctx->failover_enabled && !ctx->vote_authority ) ) own_rank = USHORT_MAX; ag_pool_advance_epoch ( ctx->pool, epoch_info, own_rank, msg->start_slot ); ag_votor_advance_epoch( ctx->votor, ctx->ns_per_slot, own_rank, msg->start_slot, NULL ); update_own_bls_key( ctx ); @@ -1566,6 +1871,7 @@ handle_replay( fd_votor_tile_t * ctx, fd_replay_slot_completed_t const * slot_completed = &replay->slot_completed; ag_block_id_t block_id = ag_block_id( slot_completed->slot, slot_completed->block_id.uc ); ag_block_id_t parent_block_id = ag_block_id( slot_completed->parent_slot, slot_completed->parent_block_id.uc ); + if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_slot_completed( ctx, slot_completed ); if( FD_UNLIKELY( ag_pool_finalized_slot( ctx->pool )==ULONG_MAX ) ) { ctx->boot_block_id = block_id; @@ -1844,6 +2150,7 @@ during_housekeeping( fd_votor_tile_t * ctx ) { ulong next_window = ag_first_slot_in_window( ctx->highest_parent_ready_slot )+AG_SLOTS_PER_WINDOW; ctx->next_leader_slot = fd_multi_epoch_leaders_get_next_slot( ctx->mleaders, fd_ulong_max( fd_ulong_max( ctx->curr_epoch_slot, next_window ), ctx->vote_history_slot ), &ctx->id_key ); + if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_switched( ctx ); fd_keyswitch_state( ctx->id_keyswitch, FD_KEYSWITCH_STATE_COMPLETED ); } } @@ -2076,7 +2383,8 @@ after_credit( fd_votor_tile_t * ctx, ctx->next_leader_slot = fd_multi_epoch_leaders_get_next_slot( ctx->mleaders, ag_first_slot_in_window( ctx->next_leader_slot )+AG_SLOTS_PER_WINDOW, &ctx->id_key ); } ag_block_id_t parent = { .slot = ULONG_MAX }; /* never will be leader, schedule exhausted, or the pool has not granted parent ready yet */ - if( FD_LIKELY( ctx->next_leader_slot!=ULONG_MAX ) ) parent = ag_pool_wait_for_parent_ready( ctx->pool, ctx->next_leader_slot ); + if( FD_LIKELY( ctx->next_leader_slot!=ULONG_MAX && ( !ctx->failover_enabled || ctx->vote_authority ) ) ) + parent = ag_pool_wait_for_parent_ready( ctx->pool, ctx->next_leader_slot ); if( FD_UNLIKELY( parent.slot!=ULONG_MAX && !ctx->halt_signing ) ) { ctx->highest_vote_or_lead_slot = fd_ulong_max( ctx->highest_vote_or_lead_slot, ag_first_slot_in_window( ctx->next_leader_slot )+AG_SLOTS_PER_WINDOW-1UL ); if( FD_LIKELY( ctx->vote_history_fd_dir!=-1 ) ) write_vote_history( ctx ); /* a restart does not lead the window again */ @@ -2088,6 +2396,7 @@ after_credit( fd_votor_tile_t * ctx, ulong reward_slot = fd_ulong_sat_sub( ctx->next_leader_slot, FD_NUM_SLOTS_FOR_REWARD ); for( ulong i=0UL; ifailover_enabled ) ) failover_after_leader( ctx, stem ); ctx->next_leader_slot = fd_multi_epoch_leaders_get_next_slot( ctx->mleaders, ag_first_slot_in_window( ctx->next_leader_slot )+AG_SLOTS_PER_WINDOW, &ctx->id_key ); *charge_busy = 1; } @@ -2265,7 +2574,8 @@ after_credit( fd_votor_tile_t * ctx, ag_epoch_info_t const * epoch_info = fd_ptr_if( vote_slot>=ctx->next_epoch_slot, ctx->next_epoch_info, fd_ptr_if( vote_slot>=ctx->curr_epoch_slot, ctx->curr_epoch_info, ctx->prev_epoch_info ) ); ulong rank = ag_vote_rank( &ctx->scratch.vote ); ctx->highest_vote_or_lead_slot = fd_ulong_max( ctx->highest_vote_or_lead_slot, vote_slot ); - if( FD_LIKELY( epoch_info && rankvalidator_cnt && vote_slot>=ctx->vote_history_slot && vote_slot>=ctx->catchup_vote_slot && !blst_p2_affine_is_inf( ag_vote_sig( &ctx->scratch.vote ) ) ) ) { + int sent = epoch_info && rankvalidator_cnt && vote_slot>=ctx->vote_history_slot && vote_slot>=ctx->catchup_vote_slot && !blst_p2_affine_is_inf( ag_vote_sig( &ctx->scratch.vote ) ); + if( FD_LIKELY( sent ) ) { if( FD_LIKELY( ctx->vote_history_fd_dir!=-1 ) ) write_vote_history( ctx ); ulong ser_sz = ag_vote_ser( &ctx->scratch.vote, ctx->scratch.ser ); long broadcast_start_time = fd_clock_tile_now( ctx->clock ); @@ -2319,6 +2629,7 @@ after_credit( fd_votor_tile_t * ctx, report_alpenglow_vote( ctx, NULL, &ctx->scratch.vote, ctx->scratch.ser[ 1 ], result, quorum_reached, FD_EVENT_ALPENGLOW_VOTE_REASON_BLOCK_REPLAYED+reason, aggregation_start_time, broadcast_start_time, reward_leader ); if( FD_UNLIKELY( reward_leader && ( !reward_leader->tx_conn || reward_leader->tx_conn->state!=FD_QUIC_CONN_STATE_ACTIVE ) ) ) report_alpenglow_vote( ctx, NULL, &ctx->scratch.vote, ctx->scratch.ser[ 1 ], result, 0, FD_EVENT_ALPENGLOW_VOTE_REASON_REWARD_NO_CONNECTION, 0L, 0L, reward_leader ); } + if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_after_vote( ctx, stem, vote_slot, sent ); *opt_poll_in = 0; *charge_busy = 1; } @@ -2383,6 +2694,12 @@ before_frag( fd_votor_tile_t * ctx, if( FD_UNLIKELY( !ctx->curr_epoch_info ) ) return -1; ctx->replay_in_seq = seq+1UL; return 0; + case IN_KIND_FAILOV: + /* Drain votes and pool events produced under the boot identity + before changing the history or removing its rank. */ + if( FD_UNLIKELY( ctx->normal_startup && ctx->init && + ( ag_votor_metrics( ctx->votor ).vote_events_cnt || ag_pool_metrics( ctx->pool ).pool_events_cnt ) ) ) return -1; + return 0; default: FD_LOG_ERR(( "unexpected in_kind %d", ctx->in_kind[ in_idx ] )); } @@ -2414,6 +2731,9 @@ during_frag( fd_votor_tile_t * ctx, case IN_KIND_REPLAY: handle_replay( ctx, sig, fd_chunk_to_laddr_const( ctx->in[ in_idx ].mem, chunk ) ); break; + case IN_KIND_FAILOV: + failover_adopt_copy( ctx, in_idx, chunk, sz ); + break; default: FD_LOG_ERR(( "unexpected in_kind %d", ctx->in_kind[ in_idx ] )); } @@ -2466,6 +2786,10 @@ after_frag( fd_votor_tile_t * ctx, } case IN_KIND_REPLAY: /* reliable link, handled in during_frag */ + if( FD_UNLIKELY( ctx->failover_enabled && sig==REPLAY_SIG_SLOT_COMPLETED ) ) publish_hist( ctx, stem, 0 ); /* keeps the failover deadlines ticking */ + break; + case IN_KIND_FAILOV: + failover_adopt_answer( ctx, sig, stem ); break; default: FD_LOG_ERR(( "unexpected in_kind %d", ctx->in_kind[ in_idx ] )); @@ -2515,16 +2839,20 @@ privileged_init( fd_topo_t const * topo, if( FD_UNLIKELY( -1==faccessat( ctx->vote_history_fd_dir, bin, F_OK, 0 ) ) ) { if( FD_UNLIKELY( errno!=ENOENT ) ) FD_LOG_ERR(( "faccessat(`%s/%s`) failed (%i-%s)", tile->votor.vote_history_path, bin, errno, fd_io_strerror( errno ) )); - DIR * dir = opendir( tile->votor.vote_history_path ); - if( FD_UNLIKELY( !dir ) ) FD_LOG_ERR(( "opendir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); - errno = 0; - for( struct dirent * ent; (ent = readdir( dir )); ) { - if( FD_LIKELY( strncmp( ent->d_name, "vote_history-", sizeof("vote_history-")-1UL ) ) ) continue; - FD_LOG_ERR(( "Booting as %s, but detected %s in %s, which is missing %s. Please either copy %s into %s or remove the vote_history-* files there if you wish to boot without a vote history file.", - id_key_b58, ent->d_name, tile->votor.vote_history_path, bin, bin, tile->votor.vote_history_path )); + /* A generated passive identity has never voted. Keep existing + voting identities' files, including their backups, untouched. */ + if( !tile->votor.failover_standby || strcmp( tile->votor.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) { + DIR * dir = opendir( tile->votor.vote_history_path ); + if( FD_UNLIKELY( !dir ) ) FD_LOG_ERR(( "opendir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); + errno = 0; + for( struct dirent * ent; (ent = readdir( dir )); ) { + if( FD_LIKELY( strncmp( ent->d_name, "vote_history-", sizeof("vote_history-")-1UL ) ) ) continue; + FD_LOG_ERR(( "Booting as %s, but detected %s in %s, which is missing %s. Please either copy %s into %s or remove the vote_history-* files there if you wish to boot without a vote history file.", + id_key_b58, ent->d_name, tile->votor.vote_history_path, bin, bin, tile->votor.vote_history_path )); + } + if( FD_UNLIKELY( errno ) ) FD_LOG_ERR(( "readdir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( closedir( dir ) ) ) FD_LOG_ERR(( "closedir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); } - if( FD_UNLIKELY( errno ) ) FD_LOG_ERR(( "readdir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); - if( FD_UNLIKELY( closedir( dir ) ) ) FD_LOG_ERR(( "closedir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); FD_LOG_WARNING(( "vote history file `%s/%s` does not exist, creating it. This identity is assumed to have never voted.", tile->votor.vote_history_path, bin )); } ctx->vote_history_slot = read_vote_history( ctx, tile->votor.vote_history_path, bin ); @@ -2676,6 +3004,7 @@ unprivileged_init( fd_topo_t const * topo, } else if( FD_LIKELY( !strcmp( link->name, "replay_slot" ) ) ) ctx->in_kind[ i ] = IN_KIND_REPLAY; else if( FD_LIKELY( !strcmp( link->name, "sign_votor" ) ) ) ctx->in_kind[ i ] = IN_KIND_SIGN; + else if( FD_LIKELY( !strcmp( link->name, "failov_votor" ) ) ) ctx->in_kind[ i ] = IN_KIND_FAILOV; else FD_LOG_ERR(( "votor tile has unexpected input link %lu %s", i, link->name )); if( FD_LIKELY( link->mtu ) ) ctx->in[ i ].mem = topo->workspaces[ topo->objs[ link->dcache_obj_id ].wksp_id ].wksp; @@ -2697,6 +3026,8 @@ unprivileged_init( fd_topo_t const * topo, ctx->net_out_wmark = fd_dcache_compact_wmark ( ctx->net_out_mem, net_out->dcache, net_out->mtu ); ctx->net_out_chunk = ctx->net_out_chunk0; + failover_init( ctx, topo, tile ); + ulong sign_in_idx = fd_topo_find_tile_in_link ( topo, tile, "sign_votor", tile->kind_id ); ulong sign_out_idx = fd_topo_find_tile_out_link( topo, tile, "votor_sign", tile->kind_id ); FD_TEST( sign_in_idx !=ULONG_MAX ); @@ -2827,7 +3158,9 @@ populate_allowed_fds( fd_topo_t const * topo, return out_cnt; } -#define STEM_BURST FD_VOTOR_OUT_BURST /* votor_out only; EXCLUDES VOTOR_NET (has no reliable consumers) */ +/* votor_out has the largest burst. votor_hist publishes at most two + frames per iteration and votor_failov publishes at most one. */ +#define STEM_BURST FD_VOTOR_OUT_BURST #define STEM_LAZY (128L*3000L) #define STEM_CALLBACK_CONTEXT_TYPE fd_votor_tile_t diff --git a/src/discof/votor/fd_votor_tile.h b/src/discof/votor/fd_votor_tile.h index 9eac4d00f65..1704f9e5437 100644 --- a/src/discof/votor/fd_votor_tile.h +++ b/src/discof/votor/fd_votor_tile.h @@ -2,6 +2,7 @@ #define HEADER_fd_src_discof_votor_fd_votor_tile_h #include "../../ballet/bls/fd_bls.h" +#include "../../choreo/votor/ag_hist.h" #include "../../disco/topo/fd_topo.h" #include "../../waltz/quic/tls/fd_quic_tls.h" @@ -126,6 +127,43 @@ union fd_votor_msg { }; typedef union fd_votor_msg fd_votor_msg_t; +/* fd_votor_hist_msg is one frame on the votor_hist link to the failover + tile, with our replay slots and our own vote history. */ + +#define FD_VOTOR_HIST_SIG (0UL) + +struct fd_votor_hist_msg { + fd_pubkey_t identity; /* identity installed when this history was published */ + ulong replay_slot; /* highest slot replay completed, ULONG_MAX before any */ + ulong root_slot; /* replay's root at the last completed slot, ULONG_MAX before any */ + ulong vote_slot; /* the history's tip, ULONG_MAX while it is empty */ + int has_vote; /* we sent a vote just before this frame, so vote_slot is a vote we signed */ + int truncated; /* the export dropped old windows and lifted the anchor */ + ag_hist_t hist; +}; +typedef struct fd_votor_hist_msg fd_votor_hist_msg_t; + +/* Adoption of a peer's vote history. */ + +#define FD_VOTOR_ADOPT_EMPTY_SZ (8UL) /* an empty request, a slot as a ulong */ + +#define FD_VOTOR_ADOPT_SUCCESS (0UL) +#define FD_VOTOR_ADOPT_ERR_DECODE (1UL) +#define FD_VOTOR_ADOPT_ERR_INVALID (2UL) +#define FD_VOTOR_ADOPT_ERR_UNREPLAYED_ROOT (3UL) /* the history's finality anchor is past our replayed slots */ +#define FD_VOTOR_ADOPT_ERR_UNREPLAYED (4UL) /* consensus is not up yet, ask again as replay moves */ +#define FD_VOTOR_ADOPT_ERR_STALE (5UL) /* older than the votes this identity sent from here */ +#define FD_VOTOR_ADOPT_ERR_CAPACITY (6UL) /* retained local and imported slots exceed capacity */ +#define FD_VOTOR_ADOPT_RESULT_CNT (7UL) + +struct fd_votor_adopt_result { + ulong result; + ulong root; /* our finality anchor after the adoption */ + ulong vote_slot; /* the adopted history's tip, ULONG_MAX for an empty one */ + ulong vote_bound; /* we cast no vote at or below this slot, ULONG_MAX when there is none */ +}; +typedef struct fd_votor_adopt_result fd_votor_adopt_result_t; + extern fd_topo_run_tile_t fd_tile_votor; #endif /* HEADER_fd_src_discof_votor_fd_votor_tile_h */ diff --git a/src/discof/votor/test_votor_tile.c b/src/discof/votor/test_votor_tile.c index 49ec2122015..9dd9fd47625 100644 --- a/src/discof/votor/test_votor_tile.c +++ b/src/discof/votor/test_votor_tile.c @@ -1742,6 +1742,42 @@ test_gossip_before_stake( void ) { test_ctx_delete( &ctx ); } +/* A tile ranked 0 in a three-voter epoch with consensus up at slot 0, + failover on, booted under its own key, for the history tests. */ + +static ag_epoch_info_t * +hist_setup( fd_votor_tile_t * ctx ) { + memset( ctx, 0, sizeof(fd_votor_tile_t) ); + ctx->vote_history_fd_dir = -1; + fd_vote_stake_weight_t stakes[ TEST_VOTER_MAX ]; + build_stakes( stakes, 3UL, 10UL ); + ag_epoch_info_t * epoch_info = rank_voters( &epoch_info_mem, stakes, 3UL, NULL, NULL ); + memcpy( ctx->id_key.uc, epoch_info->validators[0].id_key, sizeof(fd_pubkey_t) ); + ctx->boot_id_key = ctx->id_key; + ctx->curr_epoch_info = epoch_info; + ctx->failover_enabled = 1; + ctx->hist_out_idx = ULONG_MAX; + ctx->failov_out_idx = ULONG_MAX; + ctx->last_leader_slot = ULONG_MAX; + ctx->adopted_last_leader_slot = ULONG_MAX; + ctx->last_vote_slot = ULONG_MAX; + ctx->root_slot = ULONG_MAX; + + ctx->pool = test_pool( epoch_info, 0UL ); + ctx->votor = ag_votor_join( ag_votor_new( votor_scratch, 64UL, 42UL ) ); + FD_TEST( ctx->votor ); + ag_votor_init ( ctx->votor, &(ag_block_id_t){ .slot = 0UL }, 0L, 400000000L, (ushort)1, capture_sign_bls, NULL ); + ag_votor_advance_epoch( ctx->votor, 400000000L, 0UL, 0UL, epoch_info->validators[0].bls_key ); + ctx->init = 1; + return epoch_info; +} + +static void +hist_teardown( fd_votor_tile_t * ctx ) { + ag_votor_delete( ag_votor_leave( ctx->votor ) ); + ag_pool_delete( ag_pool_leave( ctx->pool ) ); +} + /* after_credit connects queued peers once due and leaves the rest, requeues one whose backoff grew, and drops entries for peers that can no longer be connected. */ @@ -2619,6 +2655,303 @@ test_reward_certs( void ) { FD_LOG_NOTICE(( "pass: test_reward_certs" )); } +/* Test that an adopted history raises the leader floor, and a lower one + leaves it alone. */ + +static void +test_history_raises_the_leader_floor( void ) { + static fd_votor_tile_t ctx; + hist_setup( &ctx ); + + ag_hist_t hist = { .anchor = 0UL, .last_leader_slot = 12UL, .vote_bound = 17UL, .rec_cnt = 1UL }; + hist.rec[ 0 ].slot = 1UL; + hist.rec[ 0 ].flags = AG_HIST_FLAG_VOTED; + uchar req[ AG_HIST_SER_MAX ]; + ulong req_sz; + FD_TEST( !ag_hist_ser( &hist, req, sizeof(req), &req_sz ) ); + fd_votor_adopt_result_t result = failover_adopt_hist( &ctx, req, req_sz ); + FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS && result.vote_slot==1UL && result.vote_bound==17UL ); + FD_TEST( ag_votor_vote_bound( ctx.votor )==17UL ); + FD_TEST( ctx.adopted_last_leader_slot==ag_first_slot_in_window( 17UL ) ); + FD_TEST( leader_floor( &ctx )==ag_first_slot_in_window( 17UL ) ); + + hist.last_leader_slot = 4UL; + hist.vote_bound = 6UL; + FD_TEST( !ag_hist_ser( &hist, req, sizeof(req), &req_sz ) ); + result = failover_adopt_hist( &ctx, req, req_sz ); + FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS && result.vote_bound==17UL ); + FD_TEST( ctx.adopted_last_leader_slot==ag_first_slot_in_window( 17UL ) ); + + uchar empty[ FD_VOTOR_ADOPT_EMPTY_SZ ]; + FD_STORE( ulong, empty, 2UL ); + result = failover_adopt_hist( &ctx, empty, sizeof(empty) ); + FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS && result.vote_bound==17UL ); + FD_TEST( ctx.adopted_last_leader_slot==ag_first_slot_in_window( 17UL ) ); + + /* A history older than the votes this machine sent is rejected. */ + ctx.last_vote_slot = 3UL; + result = failover_adopt_hist( &ctx, req, req_sz ); + FD_TEST( result.result==FD_VOTOR_ADOPT_ERR_STALE ); + + hist_teardown( &ctx ); +} + +/* Test that the votor behaves as usual when the failover links are + absent. */ + +static void +test_failover_off( void ) { + static fd_votor_tile_t ctx; + static fd_topo_t topo[1]; + static fd_topo_tile_t tile[1]; + static auth_vtr_t auth_vtr_mem[ 1UL<validators[0].id_key, sizeof(fd_pubkey_t) ); + failover_init( &ctx, topo, tile ); + FD_TEST( !ctx.failover_enabled && ctx.vote_authority ); + FD_TEST( ctx.hist_out_idx==ULONG_MAX && ctx.failov_out_idx==ULONG_MAX ); + + ag_bls_key_t bls_key; memcpy( bls_key, epoch_info->validators[0].bls_key, sizeof(ag_bls_key_t) ); + init_keys( &ctx, auth_vtr_mem, &bls_key, 1UL ); + ctx.curr_epoch_info = epoch_info; + ctx.next_epoch_slot = ULONG_MAX; + ctx.next_leader_slot = ULONG_MAX; + ctx.pool = test_pool( epoch_info, 0UL ); + ctx.votor = ag_votor_join( ag_votor_new( votor_scratch, 64UL, 42UL ) ); + FD_TEST( ctx.votor ); + ag_votor_init ( ctx.votor, &(ag_block_id_t){ .slot = 0UL }, 0L, 400000000L, (ushort)1, capture_sign_bls, NULL ); + ag_votor_advance_epoch( ctx.votor, 400000000L, 0UL, 0UL, NULL ); /* no key until update_own_bls_key picks ours */ + update_own_rank( &ctx ); + update_own_bls_key( &ctx ); + ctx.init = 1; + + /* We lead every slot of epoch 0. */ + + ctx.mleaders = fd_multi_epoch_leaders_join( fd_multi_epoch_leaders_new( mleaders_mem ) ); + ctx.peers = peers_join( peers_new( peers_mem, PEERS_LG_SLOT_CNT, 42UL ) ); + ctx.reconn_prq = reconn_prq_join( reconn_prq_new( reconn_prq_mem, RECONN_MAX ) ); + FD_TEST( ctx.mleaders && ctx.peers && ctx.reconn_prq ); + static uchar stake_msg_mem[ FD_STAKE_CI_STAKE_MSG_HEADER_SZ+FD_STAKE_CI_STAKE_MSG_RECORD_SZ ] __attribute__((aligned(8))); + fd_stake_weight_msg_t * stake_msg = fd_type_pun( stake_msg_mem ); + *stake_msg = (fd_stake_weight_msg_t){ .epoch = 0UL, .staked_vote_cnt = 1UL, .start_slot = 0UL, .slot_cnt = 64UL }; + *(fd_vote_stake_weight_t *)fd_type_pun( stake_msg+1 ) = (fd_vote_stake_weight_t){ .vote_key = ctx.id_key, .id_key = ctx.id_key, .stake = 10UL }; + fd_multi_epoch_leaders_stake_msg_init( ctx.mleaders, stake_msg ); + fd_multi_epoch_leaders_stake_msg_fini( ctx.mleaders ); + fd_aio_t * aio = fd_aio_join( fd_aio_new( aio_mem, NULL, drop_aio_send ) ); + ctx.quic_client = test_quic( quic_client_scratch, sizeof(quic_client_scratch), FD_QUIC_ROLE_CLIENT, &ctx, aio ); + ctx.quic_server = test_quic( quic_server_scratch, sizeof(quic_server_scratch), FD_QUIC_ROLE_SERVER, &ctx, aio ); + ctx.auth_vtr_keyswitch = fd_keyswitch_join( fd_keyswitch_new( av_keyswitch_mem, FD_KEYSWITCH_STATE_UNLOCKED ) ); + ctx.id_keyswitch = fd_keyswitch_join( fd_keyswitch_new( id_keyswitch_mem, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.auth_vtr_keyswitch && ctx.id_keyswitch ); + fd_clock_tile_init( ctx.clock ); + ctx.vote_history_fd_dir = -1; + for( ulong i=0UL; ibytes, ctx.id_key.uc, sizeof(fd_pubkey_t) ); + ctx.id_keyswitch->result = 77UL; + fd_keyswitch_state( ctx.id_keyswitch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( ctx.id_keyswitch->state==FD_KEYSWITCH_STATE_COMPLETED ); + FD_TEST( ctx.next_leader_slot==8UL && ctx.vote_authority ); + FD_TEST( ctx.id_keyswitch->result==77UL ); + + fd_quic_delete( fd_quic_leave( fd_quic_fini( ctx.quic_client ) ) ); + fd_quic_delete( fd_quic_leave( fd_quic_fini( ctx.quic_server ) ) ); + reconn_prq_delete( reconn_prq_leave( ctx.reconn_prq ) ); + hist_teardown( &ctx ); +} + +/* Test that under failover the staked key votes only after an adoption or + set-identity, and the junk key never does. */ + +static void +operator_switch( int operator, + ulong new_rank, + int expect_votes ) { + static fd_votor_tile_t ctx; + static auth_vtr_t auth_vtr_mem[ 1UL<validators[ new_rank ].bls_key, sizeof(ag_bls_key_t) ); + init_keys( &ctx, auth_vtr_mem, &bls_key, 1UL ); /* the new identity's key, as the resume loads it */ + ctx.id_keyswitch = fd_keyswitch_join( fd_keyswitch_new( id_keyswitch_mem, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); + FD_TEST( ctx.id_keyswitch ); + ctx.id_keyswitch->operator = (ulong)operator; + ctx.hist_seq = 77UL; + + memcpy( ctx.id_key.uc, epoch_info->validators[ new_rank ].id_key, sizeof(fd_pubkey_t) ); + failover_switched( &ctx ); + FD_TEST( ctx.vote_authority==expect_votes && !ctx.failover_hist_adopted ); + FD_TEST( ctx.id_keyswitch->result==77UL ); + update_own_rank( &ctx ); + update_own_bls_key( &ctx ); + + ag_block_info_t block = {0}; + memset( block.hash, 1, sizeof(ag_block_hash_t) ); + ag_votor_process_replay( ctx.votor, 1UL, &block ); + ag_vote_t vote; + uchar reason; + FD_TEST( ag_votor_poll_vote( ctx.votor, &vote, &reason )==expect_votes ); + if( expect_votes ) FD_TEST( ag_vote_rank( &vote )==new_rank ); + + hist_teardown( &ctx ); +} + +/* With failover on, the end of the switch applies the failover rule and + leaves the history watermark for the admin tile. */ + +static void +test_switch_runs_the_failover_rule( void ) { + static fd_votor_tile_t ctx; + static auth_vtr_t auth_vtr_mem[ 1UL<validators[0].bls_key, sizeof(ag_bls_key_t) ); + init_keys( &ctx, auth_vtr_mem, &bls_key, 1UL ); + ctx.vote_authority = 1; + ctx.hist_seq = 77UL; + ctx.mleaders = fd_multi_epoch_leaders_join( fd_multi_epoch_leaders_new( mleaders_mem ) ); + ctx.peers = peers_join( peers_new( peers_mem, PEERS_LG_SLOT_CNT, 42UL ) ); + FD_TEST( ctx.mleaders && ctx.peers ); + fd_aio_t * aio = fd_aio_join( fd_aio_new( aio_mem, NULL, drop_aio_send ) ); + ctx.quic_client = test_quic( quic_client_scratch, sizeof(quic_client_scratch), FD_QUIC_ROLE_CLIENT, &ctx, aio ); + ctx.quic_server = test_quic( quic_server_scratch, sizeof(quic_server_scratch), FD_QUIC_ROLE_SERVER, &ctx, aio ); + ctx.auth_vtr_keyswitch = fd_keyswitch_join( fd_keyswitch_new( av_keyswitch_mem, FD_KEYSWITCH_STATE_UNLOCKED ) ); + ctx.id_keyswitch = fd_keyswitch_join( fd_keyswitch_new( id_keyswitch_mem, FD_KEYSWITCH_STATE_UNLOCKED ) ); + FD_TEST( ctx.auth_vtr_keyswitch && ctx.id_keyswitch ); + fd_clock_tile_init( ctx.clock ); + + /* A failover switch to the junk key we booted under, no adoption. */ + memcpy( ctx.id_keyswitch->bytes, ctx.boot_id_key.uc, sizeof(fd_pubkey_t) ); + fd_keyswitch_state( ctx.id_keyswitch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); + during_housekeeping( &ctx ); + FD_TEST( ctx.id_keyswitch->state==FD_KEYSWITCH_STATE_COMPLETED ); + FD_TEST( !ctx.vote_authority && ctx.id_keyswitch->result==77UL ); + + hist_teardown( &ctx ); +} + +static void +test_operator_switch_votes( void ) { + operator_switch( 0, 1UL, 0 ); /* staked, no adoption */ + operator_switch( 1, 1UL, 1 ); /* staked, by set-identity */ + operator_switch( 1, 0UL, 0 ); /* back to the junk key, by set-identity */ +} + +/* Test that an adoption is rejected while this machine votes and taken + once it stands by again. */ + +static void +test_adopt_refused_while_voting( void ) { + static fd_votor_tile_t ctx; + hist_setup( &ctx ); + ctx.vote_authority = 1; + + static uchar reply_mem[ 512 ] __attribute__((aligned(FD_CHUNK_SZ))); + static fd_frag_meta_t mcache[ 8 ]; + fd_frag_meta_t * mcaches[] = { mcache }; + ulong seqs[] = { 0UL }, depths[] = { 8UL }; + ulong cr_avail = 64UL, min_cr_avail = 64UL; + int reliable = 0; + fd_stem_context_t stem[ 1 ] = {{ + .mcaches=mcaches, .seqs=seqs, .depths=depths, + .cr_avail=&cr_avail, .min_cr_avail=&min_cr_avail, + .cr_decrement_amount=1UL, .out_reliable=&reliable + }}; + ctx.failov_out_idx = 0UL; + ctx.failov_out_mem = reply_mem; + ctx.failov_out_chunk0 = 0UL; + ctx.failov_out_wmark = 4UL; + ctx.failov_out_chunk = 0UL; + ctx.in_kind[ 0 ] = IN_KIND_FAILOV; + FD_STORE( ulong, ctx.adopt_req, 9UL ); /* empty history, no vote at or below slot 9 */ + ctx.adopt_req_sz = FD_VOTOR_ADOPT_EMPTY_SZ; + + after_frag( &ctx, 0UL, 0UL, 901UL, FD_VOTOR_ADOPT_EMPTY_SZ, 0UL, 0UL, stem ); + fd_votor_adopt_result_t const * reply = fd_chunk_to_laddr_const( reply_mem, mcache[0].chunk ); + FD_TEST( seqs[0]==1UL && mcache[0].sig==901UL ); + FD_TEST( reply->result==FD_VOTOR_ADOPT_ERR_INVALID && !ctx.failover_hist_adopted ); + FD_TEST( ag_votor_vote_bound( ctx.votor )==ULONG_MAX ); + + /* A client retains its old authority if the imported slots cannot fit. */ + ctx.normal_startup = 1; + ag_hist_t hist = { .anchor = 12UL, .last_leader_slot = 80UL, .vote_bound = 83UL, .rec_cnt = AG_HIST_MAX }; + for( ulong i=0UL; iresult==FD_VOTOR_ADOPT_ERR_CAPACITY && !ctx.failover_hist_adopted ); + FD_TEST( ctx.normal_startup && ctx.vote_authority ); + FD_TEST( ctx.adopted_last_leader_slot==ULONG_MAX ); + FD_TEST( ag_votor_vote_bound( ctx.votor )==ULONG_MAX ); + FD_TEST( ag_votor_metrics( ctx.votor ).highest_final_cert_slot==0UL ); + FD_TEST( ag_votor_metrics( ctx.votor ).slot_state_pool_free==free ); + + ctx.normal_startup = 0; + ctx.vote_authority = 0; + FD_STORE( ulong, ctx.adopt_req, 9UL ); + ctx.adopt_req_sz = FD_VOTOR_ADOPT_EMPTY_SZ; + after_frag( &ctx, 0UL, 2UL, 903UL, FD_VOTOR_ADOPT_EMPTY_SZ, 0UL, 0UL, stem ); + reply = fd_chunk_to_laddr_const( reply_mem, mcache[2].chunk ); + FD_TEST( seqs[0]==3UL && mcache[2].sig==903UL ); + FD_TEST( reply->result==FD_VOTOR_ADOPT_SUCCESS && ctx.failover_hist_adopted ); + FD_TEST( ag_votor_vote_bound( ctx.votor )==9UL ); + + hist_teardown( &ctx ); +} + int main( int argc, char ** argv ) { @@ -2663,6 +2996,11 @@ main( int argc, test_write_vote_history(); test_boot_vote_history_floor(); test_reward_certs(); + test_history_raises_the_leader_floor(); + test_failover_off(); + test_switch_runs_the_failover_rule(); + test_operator_switch_votes(); + test_adopt_refused_while_voting(); FD_LOG_NOTICE(( "pass" )); fd_halt(); From 49b1f5388f93fa9d31609f127514c9b818652e5b Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Wed, 21 Oct 2026 21:39:06 +0000 Subject: [PATCH 16/23] failover: add operator guidance to the duplicate instance abort --- src/discof/gossip/fd_gossvf_tile.c | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/discof/gossip/fd_gossvf_tile.c b/src/discof/gossip/fd_gossvf_tile.c index ad75b7136db..db311cc291d 100644 --- a/src/discof/gossip/fd_gossvf_tile.c +++ b/src/discof/gossip/fd_gossvf_tile.c @@ -618,14 +618,19 @@ check_duplicate_instance( fd_gossvf_tile_ctx_t * ctx, FD_LOG_CRIT(( "unexpected message tag %u", view->tag )); } + long our_creation_nanos = ctx->instance_creation_wallclock_nanos; for( ulong i=0UL; itag!=FD_GOSSIP_VALUE_CONTACT_INFO ) ) continue; - if( FD_LIKELY( ctx->instance_creation_wallclock_nanos>=FD_MICRO_TO_NANOSEC( value->contact_info->outset ) ) ) continue; - if( FD_LIKELY( memcmp( ctx->identity_pubkey->uc, value->origin, 32UL ) ) ) continue; - - FD_LOG_ERR(( "duplicate running instances of the same validator node, our timestamp: %ldns their timestamp: %ldns", ctx->instance_creation_wallclock_nanos, FD_MICRO_TO_NANOSEC( value->contact_info->outset ) )); + long their_creation_nanos = FD_MICRO_TO_NANOSEC( value->contact_info->outset ); + if( FD_UNLIKELY( their_creation_nanos>our_creation_nanos && + !memcmp( ctx->identity_pubkey->uc, value->origin, 32UL ) ) ) { + FD_LOG_ERR(( "duplicate running instances of the same validator node, our timestamp: %ldns their timestamp: %ldns. " + "If the newer instance is a failover peer booted without `[failover.enabled]`, stop it and enable failover " + "there, then run `firedancer failover promote --force` on the machine that should hold the identity.", + our_creation_nanos, their_creation_nanos )); + } } } From 07ccfce18869414cce2a7c0d832a666eed09f747 Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Fri, 23 Oct 2026 15:05:13 +0000 Subject: [PATCH 17/23] failover: adapt admin and config tests to Tower removal --- src/app/firedancer/topology.c | 1 - src/app/shared/fd_config_json.c | 2 +- src/app/shared/test_config_parse.c | 18 +++++++----------- src/discof/admin/fd_admin_tile.c | 13 ++++++------- src/discof/admin/test_admin_tile.c | 4 ++-- 5 files changed, 16 insertions(+), 22 deletions(-) diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index bef052ed03e..54796b0d583 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -414,7 +414,6 @@ fd_topo_initialize( config_t * config ) { fd_topob_wksp( topo, "sign_failov" ); } - fd_topob_wksp( topo, "execrp_replay" ); fd_topob_wksp( topo, "admin_replay" ); if( FD_UNLIKELY( failover_enabled ) ) { diff --git a/src/app/shared/fd_config_json.c b/src/app/shared/fd_config_json.c index f1259e33400..b8ecc42384f 100644 --- a/src/app/shared/fd_config_json.c +++ b/src/app/shared/fd_config_json.c @@ -14,7 +14,7 @@ lists below. */ /* topo.boot_identity and sign.identity_key_fd are derived startup state, not operator configuration. */ -FD_STATIC_ASSERT( sizeof(fd_config_t)==24458368UL, update_fd_config_to_json_for_the_layout_change ); +FD_STATIC_ASSERT( sizeof(fd_config_t)==25509048UL, update_fd_config_to_json_for_the_layout_change ); #define REDACTED "[redacted]" diff --git a/src/app/shared/test_config_parse.c b/src/app/shared/test_config_parse.c index 7b50e547cfc..d917fbc0379 100644 --- a/src/app/shared/test_config_parse.c +++ b/src/app/shared/test_config_parse.c @@ -112,13 +112,11 @@ failover_is_valid( config_t * config, int enabled, char const * listen_address, ushort listen_port, - char const * vote_account, - int alpenglow ) { + char const * vote_account ) { int pid = fork(); FD_TEST( pid>=0 ); if( FD_UNLIKELY( !pid ) ) { config->firedancer.development.genesis.max_file_size_mib = 4055UL; - config->firedancer.consensus.alpenglow = alpenglow; config->firedancer.failover.enabled = enabled; config->firedancer.failover.listen_port = listen_port; fd_cstr_ncpy( config->firedancer.failover.listen_address, listen_address, sizeof(config->firedancer.failover.listen_address) ); @@ -237,14 +235,12 @@ main( int argc, FD_TEST( genesis_max_file_size_is_valid( config, 4055UL ) ); FD_TEST( !genesis_max_file_size_is_valid( config, 4056UL ) ); - FD_TEST( failover_is_valid( config, 1, "0.0.0.0", 8010, "vote.json", 0 ) ); - FD_TEST( failover_is_valid( config, 1, "10.0.0.1", 8010, "vote.json", 0 ) ); - FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 0, "vote.json", 0 ) ); - FD_TEST( !failover_is_valid( config, 1, "localhost", 8010, "vote.json", 0 ) ); - FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 8010, "", 0 ) ); - FD_TEST( failover_is_valid( config, 1, "0.0.0.0", 8010, "vote.json", 1 ) ); - FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 0, "vote.json", 1 ) ); - FD_TEST( failover_is_valid( config, 0, "", 0, "", 1 ) ); + FD_TEST( failover_is_valid( config, 1, "0.0.0.0", 8010, "vote.json" ) ); + FD_TEST( failover_is_valid( config, 1, "10.0.0.1", 8010, "vote.json" ) ); + FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 0, "vote.json" ) ); + FD_TEST( !failover_is_valid( config, 1, "localhost", 8010, "vote.json" ) ); + FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 8010, "" ) ); + FD_TEST( failover_is_valid( config, 0, "", 0, "" ) ); /* Ensure we can selectively override a field */ diff --git a/src/discof/admin/fd_admin_tile.c b/src/discof/admin/fd_admin_tile.c index bc94b9e5929..5d0786d2278 100644 --- a/src/discof/admin/fd_admin_tile.c +++ b/src/discof/admin/fd_admin_tile.c @@ -700,7 +700,7 @@ failover_operator_begin( fd_admin_tile_ctx_t * ctx, fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, topo->tiles[ i ].id_keyswitch_obj_id ); sign->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR; } - find_identity_keyswitch( ctx, ctx->voter_name )->operator = 1UL; + find_identity_keyswitch( ctx, "votor" )->operator = 1UL; ctx->failover_pending = 1; } @@ -797,7 +797,7 @@ set_identity( fd_admin_tile_ctx_t * ctx, failover_operator_begin( ctx, req->keypair+32UL ); } - if( ctx->failov_out_idx!=ULONG_MAX ) find_identity_keyswitch( ctx, ctx->voter_name )->operator = 1UL; + if( ctx->failov_out_idx!=ULONG_MAX ) find_identity_keyswitch( ctx, "votor" )->operator = 1UL; ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; ulong identity_outset = (ulong)fd_log_wallclock(); @@ -860,7 +860,7 @@ get_identity( fd_admin_tile_ctx_t * ctx, done carefully in order to prevent votes being generated with an authorized voter that the sign tile is not yet aware of. The authorized voter must be added to the sign tile before it is added - to the voter tile (the tower/votor tile). All transitions must be + to the votor tile. All transitions must be linear and in forward order. */ /* State 0: UNLOCKED @@ -1172,8 +1172,7 @@ snapshot_create_response( fd_admin_tile_ctx_t * ctx, /* Removing all authorized voters from the validator is the inverse of add-authorized-voter, and must be done in the opposite order. When - adding, the sign tile is updated before the voter tile (the tower - tile, or the votor tile under Alpenglow) so that the voter never asks + adding, the sign tile is updated before the votor tile so that it never asks the sign tile to sign a vote with an authority index the sign tile does not yet know about. When removing, the voter tile must be cleared before the sign tiles, so that the voter stops referencing an @@ -1654,7 +1653,7 @@ failover_switch_request( fd_admin_tile_ctx_t * ctx, ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; ulong identity_outset = (ulong)fd_log_wallclock(); for(;;) { - if( FD_UNLIKELY( poll_set_identity( ctx, &state, identity_outset, keypair, NULL, 0UL ) ) ) break; + if( FD_UNLIKELY( poll_set_identity( ctx, &state, identity_outset, keypair, 0UL ) ) ) break; } ctx->failover_switch = 0; fd_memcpy( ctx->identity_pubkey, req.identity, 32UL ); @@ -1663,7 +1662,7 @@ failover_switch_request( fd_admin_tile_ctx_t * ctx, response.result = FD_FAILOVER_SWITCH_OK; response.history_watermark = find_identity_keyswitch( ctx, "votor" )->result; report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS ); - FD_LOG_NOTICE(( "failover: every tile switched to `%s`, the %s halted signing at sequence %lu", new_identity, ctx->voter_name, response.history_watermark )); + FD_LOG_NOTICE(( "failover: every tile switched to `%s`, votor halted signing at sequence %lu", new_identity, response.history_watermark )); } else { FD_LOG_WARNING(( "the failover tile asked for an identity switch but failover is off, refusing" )); } diff --git a/src/discof/admin/test_admin_tile.c b/src/discof/admin/test_admin_tile.c index 077cfdc5d92..e330f2d83af 100644 --- a/src/discof/admin/test_admin_tile.c +++ b/src/discof/admin/test_admin_tile.c @@ -305,7 +305,7 @@ test_failover_identity( void ) { setup(); FD_TEST( fd_topob_new( test_topo, "admin-test" ) ); fd_topob_wksp( test_topo, "wksp" )->wksp = NULL; - fd_keyswitch_t * tower_ks = mock_tile( "tower" ); + fd_keyswitch_t * votor_ks = mock_tile( "votor" ); fd_keyswitch_t * sign_ks = mock_tile( "sign" ); ctx.topo = test_topo; ctx.failover_enabled = 1; @@ -319,7 +319,7 @@ test_failover_identity( void ) { nothing goes to the failover tile until every tile switched. */ failover_operator_begin( &ctx, third ); FD_TEST( ctx.failover_epoch==1UL && fd_memeq( ctx.failover_pubkey, third, 32UL ) && ctx.failover_pending && !bus_seq ); - FD_TEST( sign_ks->param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR && tower_ks->operator==1UL ); + FD_TEST( sign_ks->param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR && votor_ks->operator==1UL ); fd_memcpy( ctx.identity_pubkey, third, 32UL ); failover_operator_notify( &ctx, bus_stem ); fd_failover_operator_t operator = bus_operator(); From affaf154fc09b70c12d7fd98cd1f36b29e3612ed Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Fri, 23 Oct 2026 15:23:44 +0000 Subject: [PATCH 18/23] votor: prune imported history below the startup cursor --- src/choreo/votor/ag_votor.c | 2 + src/choreo/votor/test_ag_votor_failover.c | 47 ++++++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/src/choreo/votor/ag_votor.c b/src/choreo/votor/ag_votor.c index d2e5c0c1d6c..cc724f69ad5 100644 --- a/src/choreo/votor/ag_votor.c +++ b/src/choreo/votor/ag_votor.c @@ -734,6 +734,8 @@ ag_votor_hist_adopt( ag_votor_t * self, } advance_root( self, hist->anchor ); + /* Later pruning must visit records imported below the startup cursor. */ + self->root = fd_ulong_min( self->root, first ); /* Take the exporter's vote bound as well. */ if( FD_UNLIKELY( hist->vote_bound!=ULONG_MAX ) ) ag_votor_set_vote_bound( self, hist->vote_bound ); ulong conflicts = 0UL; diff --git a/src/choreo/votor/test_ag_votor_failover.c b/src/choreo/votor/test_ag_votor_failover.c index cf5033f2481..b29a45ea2db 100644 --- a/src/choreo/votor/test_ag_votor_failover.c +++ b/src/choreo/votor/test_ag_votor_failover.c @@ -58,19 +58,27 @@ recv( ag_votor_t * votor ) { } static ag_votor_t * -setup_votor( void * mem, - long now ) { +setup_votor_at_root( void * mem, + long now, + ulong root_slot ) { create_validators(); FD_TEST( ag_votor_footprint( TEST_SLOT_MAX )<=SCRATCH_MAX ); ag_votor_t * votor = ag_votor_join( ag_votor_new( mem, TEST_SLOT_MAX, 42UL ) ); FD_TEST( votor ); ag_bls_key_t bls_key; bls_key_from_sec( bls_key, &g_sk[0] ); ag_block_id_t root = genesis_block_id(); + root.slot = root_slot; ag_votor_init ( votor, &root, now, TEST_NS_PER_SLOT, TEST_SHRED_VERSION, sec_sign_fn, &g_sk[0] ); ag_votor_advance_epoch( votor, TEST_NS_PER_SLOT, 0UL, 0UL, bls_key ); return votor; } +static ag_votor_t * +setup_votor( void * mem, + long now ) { + return setup_votor_at_root( mem, now, 0UL ); +} + static void teardown_votor( ag_votor_t * votor ) { ag_votor_delete( ag_votor_leave( votor ) ); @@ -278,6 +286,40 @@ test_adopt_capacity( void ) { FD_LOG_NOTICE(( "pass: adopt_capacity" )); } +static void +test_adopt_equal_anchor_prunes( void ) { + ag_votor_t * votor = setup_votor_at_root( scratch_a, 0L, 100UL ); + + /* A rejected import must leave the pruning cursor and all state + unchanged, even when its records start below the initial root. */ + ag_hist_t hist = { .anchor = 100UL, .last_leader_slot = ULONG_MAX, .vote_bound = ULONG_MAX, .rec_cnt = TEST_SLOT_MAX+1UL }; + for( ulong i=0UL; iroot==104UL && votor->highest_final_cert_slot==112UL ); + for( ulong slot=92UL; slot<=100UL; slot++ ) { + FD_TEST( !slot_state_map_ele_query_const( votor->slot_states->map, &slot, NULL, votor->slot_states->pool ) ); + } + FD_TEST( slot_state_pool_free( votor->slot_states->pool )==TEST_SLOT_MAX-AG_SLOTS_PER_WINDOW ); + + teardown_votor( votor ); + FD_LOG_NOTICE(( "pass: adopt_equal_anchor_prunes" )); +} + int main( int argc, char ** argv ) { @@ -287,6 +329,7 @@ main( int argc, test_mark_unsent_notar(); test_bound_travels(); test_adopt_capacity(); + test_adopt_equal_anchor_prunes(); FD_LOG_NOTICE(( "pass" )); fd_halt(); return 0; From e9b8e3af8dbc9b6ad0e44b21f8dbf851c22c428c Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Fri, 23 Oct 2026 15:23:44 +0000 Subject: [PATCH 19/23] failover: remove obsolete Alpenglow activation guidance Alpenglow is always on and Frankendancer is gone, so the failover docs, config comments and CLI errors no longer say either is required. The junk identity is generated fresh on every boot, so the docs, CLI help and startup log no longer describe it as the machine's own key. Requesting a handoff from a validator without failover enabled stays undocumented, so the docs, config comments, CLI help, errors and startup log no longer describe it. The set-identity docs now point handoffs between two Firedancer validators to failover promote instead of describing how set-identity interacts with failover. The failover promote docs also drop the notes on pre-confirmation authentication, missing gossip addresses and option combinations, and failover comments drop details that restated the code. Co-Authored-By: Claude Opus 5.5 --- book/api/cli.md | 180 +++------------------- src/app/firedancer/commands/failover.c | 28 ++-- src/app/firedancer/config/default.toml | 14 +- src/app/shared/commands/run/run.c | 4 +- src/app/shared/fd_config.c | 2 +- src/disco/keyguard/fd_sign_tile.c | 8 +- src/disco/keyguard/test_sign_tile.c | 2 +- src/disco/topo/fd_topo.h | 2 +- src/discof/admin/fd_admin_tile.c | 10 +- src/discof/admin/test_admin_tile.c | 2 +- src/discof/failover/fd_failover_channel.c | 2 +- src/discof/failover/fd_failover_tile.c | 7 +- 12 files changed, 54 insertions(+), 207 deletions(-) diff --git a/book/api/cli.md b/book/api/cli.md index b2a48e2d22a..864bbb45dbf 100644 --- a/book/api/cli.md +++ b/book/api/cli.md @@ -190,6 +190,16 @@ argument is required and must be the path to an Agave style `identity.json` keypair file. If the path is specified as `-` the key will instead be read from `stdin`. +::: tip NOTE + +Between two Firedancer validators, Firedancer supports a native +[failover](#failover-promote) mechanism that is faster and easier than +`set-identity`, and using `set-identity` between them is heavily +discouraged. Use `set-identity` primarily for failing over between Agave +and Firedancer. + +::: + It is not generally safe to call `set-identity`, as another validator might be running with the same identity, and if they both produce a block or vote concurrently, the validator may violate consensus and be @@ -219,19 +229,6 @@ the running validator will not be changed. Switching to the identity the validator is already running as is refused. -Between two Firedancer validators running failover, only use -`firedancer failover promote`, which hands over the vote history -directly. Do not use `--vote-history-file` for this, it is only for a -vote history file from a validator that does not run failover, such as Agave. - -With failover on, `set-identity` also makes the new key the identity -`failover promote` hands off, unless it is the machine's junk identity. -Only a machine running the same key can take it over, so set that -key as `[paths.identity_key]` on the other machine and restart it. -After a restart, the machine boots as standby under its junk identity -and uses `[paths.identity_key]` as the handoff target again. -The new key cannot also be one of the authorized voters. - The validator writes `vote_history-.bin` in the `[paths.vote_history]` directory before each vote and each block it produces. This is an Agave-compatible vote history file and it protects @@ -353,9 +350,6 @@ standby. The remote validator must have `[failover.enabled]` set to `true` to accept the request. Both validators must use the same vote account. -The failover commands require the full Firedancer client with -Alpenglow consensus enabled through `[consensus.alpenglow]` or `--alpenglow`. - ```sh firedancer failover promote [] [--name ] [--yes] [--force] [--address
] [--port ] ``` @@ -363,43 +357,27 @@ firedancer failover promote [] [--name ] [--yes] [--force] [--add The command attaches to the local running validator and discovers it automatically. If more than one is running, pass `--name ` to select one (see [`ps`](#ps)). It asks for confirmation when run from a -terminal unless `--yes` is given. Outside a terminal it proceeds -without a prompt. - -Before asking for confirmation, the command checks the local validator's -role. A standby with a prepared target identity and a fresh gossip -address authenticates to the remote validator in advance. This connection -does not request a handoff. Confirmation submits the handoff request, so -the prompt does not mean the remote validator is ready to hand off. +terminal unless `--yes` is given. The local validator finds the active's address through gossip. Use `--address` to give the remote validator's IPv4 address or host name instead, for example when connecting over a private network. Use -`--port` if its failover listener uses a different port from the local -validator's `[failover.listen_port]`. - -If gossip has no address for the remote validator, the command refuses -the request. Wait for its contact information or provide `--address`. -Providing an address does not bypass peer authentication. +`--port` if its failover listener does not use the default port 8010. For example, to connect to a known failover listener over a private network, run the following on the local standby. ```sh -firedancer failover promote --name standby --address 192.0.2.10 --port 8010 +firedancer failover promote --address 192.0.2.10 --port 8010 ``` -The remote validator must be reachable at that TCP address and port. If -it is stopped, its failover listener is disabled, or the connection -cannot be authenticated, the local request cannot complete the handoff. - -### Both validators have failover enabled +### Setup Set `[failover.enabled]` to `true` on both validators. Configure the same staked identity in `[paths.identity_key]` and the same vote account in `[paths.vote_account]`. Firedancer generates a fresh junk identity in memory -on each startup. The private key is never written to a file. Both validators -boot under their junk identities without voting and listen for handoff requests. +on each startup. Both validators boot under a junk identity without voting +and listen for handoff requests. Once one validator is active, run the following on the standby to request a handoff. The staked identity keypair is already loaded from the @@ -420,127 +398,34 @@ After the handoff, the old active can request the identity back with one with `--force` as described below. An optional keypair argument must match the standby's current failover target. -### Only the active has failover enabled - -Keep `[failover.enabled]` set to `true` on the active and set it to `false` -on the requester. Disabling failover does not disable voting. The -requester starts under `[paths.identity_key]` without generating a junk -identity or automatically becoming a nonvoting standby. - -For this workflow, start the requester with a junk identity in -`[paths.identity_key]` and the active's vote account in -`[paths.vote_account]`. The junk identity must differ from the active's -identity and must not be a staked node identity. This keeps the requester -nonvoting while it replays, even with authorized voter keys loaded. The -vote account must remain configured for takeover. - -If needed, create a junk key with the existing `keys` command and set -`[paths.identity_key]` to its path before starting the requester. - -```sh -firedancer keys new /path/to/standby-identity.json -firedancer run --config /path/to/standby.toml -``` - -Alpenglow validators with a vote account include a failover tile for outgoing -requests, so allow for it when assigning CPU affinity. Pass the active's -identity keypair on first use. This prepares the target without installing -it. The requester must still be using the identity it started with. The -target cannot also be an authorized voter. - -```sh -firedancer failover promote /path/to/active-identity.json --name standby -firedancer failover status --name standby -``` - -Later requests can omit the keypair. Repeating the same keypair preserves -an ongoing handoff. A different keypair is refused. After takeover, the -requester's failover listener stays disabled, so the other validator -cannot request the identity back from it. On restart, it returns to its -configured identity and forgets the prepared target, so the next request -needs the keypair again. - -If both validators have failover disabled, neither can accept a handoff -request. - -### Handoff progress and failures - -The following applies to both configurations. - -The remote validator refuses the handoff if it has no final vote history -to hand over, the local standby has not started replaying, or the standby -is more than 64 slots behind its finality anchor. It keeps its identity in -these cases. It also refuses a new request while another transition is running. Once it agrees to hand -off, it gives up the identity before the local validator adopts its -vote history. If adoption fails or stalls, the handoff can fail with -neither machine voting. - -Exit code 0 means the local validator accepted the request. It does not -mean the remote validator has agreed to hand off or that the handoff has -completed. The command prints a handoff ID. Check -[`failover status`](#failover-status) on the local validator for -`role: active` and `last handoff: , taken` with that ID. Check that the -remote validator shows `role: standby` and `action: idle`. - -The local validator refuses a new request if it is already active or -another transition is running. A local refusal exits with a non-zero -code and prints the reason to `stderr`. An accepted request can still -fail while connecting to the remote validator, authenticating it, or -adopting its vote history. These later failures do not change the -command's exit code. Check status and the logs on both validators to -determine where it stopped. - -If the command itself times out waiting for the local validator to -answer, the outcome is unknown. Check status on both machines because -the request may still complete. - -If the request pauses at its network deadline, the local validator stops -dialing. Run `failover status` on both validators before retrying. Run -`failover promote` again on the local validator to resume the -request. This also resumes an active validator that is waiting for the -remote validator to confirm completion. You can provide a new `--address` -or `--port` when resuming it. - ### Forced promotion With `--force`, the local standby takes the identity without contacting the remote validator. This is also needed for first use when both -machines are standbys. Promotion first tries eligible saved peer vote -history. If that history cannot be adopted, it falls back to empty vote -history with a slot bound that prevents voting at or below it. Check for -`role: active` and `action: idle` to confirm completion. - -Forced promotion can cancel a handoff that is waiting for the remote -validator's response. It cannot interrupt a local vote history adoption -or identity switch. Replay and adoption must still complete before the -identity can be installed. +machines are standbys. ::: warning WARNING Only use `--force` after ensuring every other machine with the identity cannot sign. It skips the checks on the other machine and can leave votes and leader windows unaccounted for by accepting incomplete or empty history. -Never force promotion on both machines at once. `--yes` only skips the -confirmation prompt. It does not imply `--force` or suppress its warning. +Never force promotion on both machines at once. ::: | Arguments | Description | |-----------------------|-------------| -| `` | Optionally provide the target identity keypair file. Required on first use when `[failover.enabled]` is `false`. | +| `` | Optionally provide the target identity keypair file. It must match the standby's current failover target. | | `--name ` | Select the validator instance to attach to if more than one is running on this host. | | `--config ` | Optionally use the configuration TOML file the validator was started with. | | `--yes` | Skip the confirmation prompt. | -| `--force` | Take the identity without cooperation from the remote validator, allowing incomplete or empty vote history. This option cannot be combined with `--address` or `--port`. | -| `--address
` | Use this IPv4 address or host name for the remote validator instead of the address found through gossip. This option cannot be combined with `--force`. | -| `--port ` | Set the remote validator's failover TCP port. The default is the local validator's `[failover.listen_port]`, which is 8010 unless configured otherwise. This option cannot be combined with `--force`. | +| `--force` | Take the identity without cooperation from the remote validator, allowing incomplete or empty vote history. | +| `--address
` | Use this IPv4 address or host name for the remote validator instead of the address found through gossip. | +| `--port ` | Set the remote validator's failover TCP port. The default is 8010. | ## `failover status` -This command prints the local running validator's failover role, current -action, peer and last handoff. It also reports what this machine knows -about another holder of the identity and the vote history a forced -promotion would adopt. Use it to check progress after -[`failover promote`](#failover-promote). +This command prints the local running validator's failover status. Use +it to check progress after [`failover promote`](#failover-promote). ```sh firedancer failover status [--name ] @@ -552,10 +437,6 @@ It exits with code 0 if status was retrieved, including when failover is disabled. Otherwise it exits with a non-zero code and prints the reason to `stderr`. -With Alpenglow enabled, `[failover.enabled]` set to `false`, and a vote -account configured, status reports `client only, listener disabled` and -shows outgoing handoff progress. - The following example shows a completed handoff. The last handoff ID matches the ID printed by `failover promote` above. The connection closes after completion, so `peer role` is unknown until another authenticated @@ -564,19 +445,6 @@ new active through gossip. <<< @/snippets/commands/failover-status.ansi -`role` describes the local validator. `peer role` describes the remote -validator only while an authenticated session is open. `last handoff` -records the latest request's outcome, so check `role` and `action` as -well when investigating a failed or paused handoff. - -`other holder` reports what the local validator knows about another -machine using the identity. Seeing none does not prove every other -machine is unable to sign. `takeover adopts` shows the vote history -source a forced promotion would currently select. `coverage floor` is -the highest slot known to have been signed locally or reported by the -remote validator. Forced promotion can proceed without history covering -that slot, as described above. - | Arguments | Description | |-----------------------|-------------| | `--name ` | Select the validator instance to attach to if more than one is running on this host. | diff --git a/src/app/firedancer/commands/failover.c b/src/app/firedancer/commands/failover.c index 39b328a303a..acc761554c0 100644 --- a/src/app/firedancer/commands/failover.c +++ b/src/app/firedancer/commands/failover.c @@ -41,9 +41,7 @@ FD_STATIC_ASSERT( SESSION_NAME_CNT==FD_FAILOVER_SESSION_CNT, session_names FD_STATIC_ASSERT( SOURCE_NAME_CNT ==FD_FAILOVER_SOURCE_CNT, source_names ); FD_STATIC_ASSERT( HANDOFF_NAME_CNT==FD_FAILOVER_HANDOFF_CNT, handoff_names ); -/* Strips key and its value. fd_env_strip_cmdline_cstr silently drops a - key given last with no value and takes a flag that follows the key as - its value, so we refuse both before we strip. */ +/* Strips key and its value. */ static char const * strip_value( int * pargc, char *** pargv, @@ -134,7 +132,7 @@ action_name( uchar action ) { return actionfailover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_PROMOTE ) { FD_LOG_STDOUT(( "This takes the staked identity on your word. No other machine may hold it or be\n" "in a promotion, including any validator running this identity without failover.\n" @@ -197,8 +194,7 @@ failover_confirm( args_t const * args ) { } /* What this machine knows of another holder of the staked identity, - from the checks that `failover promote --force` skips. It is not a go - ahead. */ + from the checks that `failover promote --force` skips. */ static char const * other_holder( ulong result ) { switch( result ) { @@ -307,7 +303,7 @@ failover_request( fd_adminctl_t * adminctl, static void __attribute__((noreturn)) fail_disabled( int status ) { if( status ) FD_LOG_ERR(( "`failover status` is unavailable because this validator has no failover command bus" )); - FD_LOG_ERR(( "the running validator has no failover client, run Alpenglow consensus with [paths.vote_account] configured" )); + FD_LOG_ERR(( "failover is not enabled on the running validator, set [failover.enabled] to true and restart it" )); } static void __attribute__((noreturn)) @@ -418,8 +414,8 @@ failover_status_args_help( fd_action_help_t * help ) { static void failover_args_help( fd_action_help_t * help ) { fd_action_help_arg( help, "", NULL, "Request takeover on this validator, or read its failover state" ); - fd_action_help_arg( help, "[]", NULL, "Path to the target identity keypair. Required on first use when\n" - "[failover.enabled] is false. Loading it does not install it" ); + fd_action_help_arg( help, "[]", NULL, "Path to the target identity keypair, which must match the\n" + "standby's current failover target" ); failover_status_args_help( help ); fd_action_help_arg( help, "--yes", NULL, "Skip the confirmation prompt. It does not authorize peer or\n" "vote-history overrides" ); @@ -454,15 +450,9 @@ action_t fd_action_failover = { .perm = failover_promote_perm, .description = "Promote the running standby validator or read its failover state", .detail = "Hands off the staked identity between two validators. A validator with\n" - "[failover] enabled boots as a standby under its junk identity and does\n" - "not vote until promoted. Both validators must use the same\n" + "[failover] enabled boots as a standby under a fresh junk identity and\n" + "does not vote until promoted. Both validators must use the same\n" "[paths.vote_account].\n" - "With [failover.enabled] false, start the requester with an unstaked junk\n" - "key in [paths.identity_key] to keep it nonvoting until takeover. Provide\n" - "the active's keypair on first use. The requester keeps its boot identity\n" - "until adoption and the identity switch complete. It does not listen for\n" - "incoming handoffs. The target must differ from the boot identity and\n" - "cannot be an authorized voter.\n" "\n" "Asks the active to hand over its identity. With --force, takes the\n" "identity without the active, for when it cannot sign. Both forms\n" diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index 89e060434f7..e382e882053 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -1991,16 +1991,10 @@ telemetry = true # history files are only for handing off an identity from a validator that # does not run failover, such as Agave. [failover] - # Whether to accept incoming handoff requests. Requires Alpenglow - # consensus, enabled by [consensus.alpenglow] or --alpenglow. When - # true, the validator boots as standby under a fresh junk identity generated - # in memory. Its private key is not written to a file. - # When false, it boots normally under [paths.identity_key]. For a - # nonvoting requester, use an unstaked junk key as [paths.identity_key] - # and the active's vote account as [paths.vote_account]. Request an - # outgoing handoff using `failover promote `. The target - # keypair is prepared without installing it and must differ from the - # boot identity. + # Whether to enable failover. When true, the validator boots as + # standby under a fresh junk identity generated in memory. Its + # private key is not written to a file. When false, it boots + # normally under [paths.identity_key]. enabled = false # The address to listen on. If enabled, by default the listener is diff --git a/src/app/shared/commands/run/run.c b/src/app/shared/commands/run/run.c index 0ad2f84ab18..d73910dcf4c 100644 --- a/src/app/shared/commands/run/run.c +++ b/src/app/shared/commands/run/run.c @@ -1085,8 +1085,8 @@ fdctl_check_configure( config_t const * config ) { fd_cpu_isolation_check_cgroup( config ); } -/* Each Sign tile gets its own one-shot pipe. Only the public key goes - into the config shared with other tiles and saved for CLI discovery. */ +/* Each Sign tile gets its own one-shot pipe for receiving the ephemeral + generated junk key. */ static void FD_FN_SENSITIVE failover_junk_prepare( config_t * config ) { if( fd_topo_find_tile( &config->topo, "failov", 0UL )==ULONG_MAX ) return; diff --git a/src/app/shared/fd_config.c b/src/app/shared/fd_config.c index f3c233ff449..01125878096 100644 --- a/src/app/shared/fd_config.c +++ b/src/app/shared/fd_config.c @@ -343,7 +343,7 @@ fd_config_fill( fd_config_t * config, replace( config->paths.base, "{user}", config->user ); replace( config->paths.base, "{name}", config->name ); - /* Under failover [paths.identity_key] is the staked key both machines + /* With failover, [paths.identity_key] is the staked key both machines share. */ if( FD_UNLIKELY( config->firedancer.failover.enabled && !strcmp( config->paths.identity_key, "" ) ) ) { FD_LOG_ERR(( "[failover.enabled] is true, so [paths.identity_key] must specify the staked identity keypair both machines share" )); diff --git a/src/disco/keyguard/fd_sign_tile.c b/src/disco/keyguard/fd_sign_tile.c index 28ce0325aa6..8738993d927 100644 --- a/src/disco/keyguard/fd_sign_tile.c +++ b/src/disco/keyguard/fd_sign_tile.c @@ -306,8 +306,7 @@ during_frag( void * _ctx, during_frag_sensitive( _ctx, in_idx, seq, sig, chunk, sz ); } -/* The junk key signs TLS. The prepared failover key signs admission - proofs for our own junk pubkey, whichever identity is installed. */ +/* The junk key signs TLS. */ static void FD_FN_SENSITIVE failover_sign( fd_sign_ctx_t * ctx, @@ -457,10 +456,7 @@ after_frag( void * _ctx, } /* failover_load_keys sets the failover keys, all NULL unless failover - is enabled. Under failover we also keep read only copies of the junk - and staked keypairs, loaded here before the sandbox, and a page for a - key that set-identity installs. We boot with the junk key as the - identity key. */ + is enabled. */ static void FD_FN_SENSITIVE failover_load_keys( fd_sign_ctx_t * ctx, diff --git a/src/disco/keyguard/test_sign_tile.c b/src/disco/keyguard/test_sign_tile.c index 23210f3acbd..8d56fbf82e5 100644 --- a/src/disco/keyguard/test_sign_tile.c +++ b/src/disco/keyguard/test_sign_tile.c @@ -296,7 +296,7 @@ test_bls_request_rejected( void ) { FD_TEST( bls_request_exit_status( FD_KEYGUARD_SIGN_TYPE_BLS|(1UL<<32), 11UL )==1 ); /* authorized voter 0, none loaded */ } -/* Under failover the sign tile switches by public key between the junk +/* With failover, the sign tile switches by public key between the junk key and the failover identity, the staked key from boot or a key set-identity installed. */ diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index 5ee3a3377e2..52c7c601255 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -246,7 +246,7 @@ struct fd_topo_tile { } admin; struct { - char identity_key_path[ PATH_MAX ]; /* this machine's junk key */ + char identity_key_path[ PATH_MAX ]; /* the boot identity, the junk key generated at boot with failover */ char staked_key_path[ PATH_MAX ]; /* we read only its public half */ char vote_account_path[ PATH_MAX ]; uint listen_addr; /* network byte order */ diff --git a/src/discof/admin/fd_admin_tile.c b/src/discof/admin/fd_admin_tile.c index 5d0786d2278..3bbebccc813 100644 --- a/src/discof/admin/fd_admin_tile.c +++ b/src/discof/admin/fd_admin_tile.c @@ -120,7 +120,7 @@ privileged_init( fd_topo_t const * topo, fd_memcpy( ctx->identity_pubkey, fd_keyload_load( tile->admin.identity_key_path, /* pubkey only: */ 1 ), 32UL ); - /* Under failover we boot on the junk key, and the key failover hands off + /* With failover, we boot on the junk key, and the key failover hands off starts as [paths.identity_key]. */ ulong failov_idx = fd_topo_find_tile( topo, "failov", 0UL ); if( FD_UNLIKELY( failov_idx!=ULONG_MAX ) ) { @@ -176,7 +176,7 @@ unprivileged_init( fd_topo_t const * topo, for( ulong i=0UL; iin_cnt; i++ ) { fd_topo_link_t const * link = &topo->links[ tile->in_link_id[ i ] ]; /* The stem numbers only the in links it polls, and we compare its - numbers with topology in link indexes, so under failover every in + numbers with topology in link indexes, so with failover every in link is polled. */ if( FD_UNLIKELY( ctx->failov_in_idx!=ULONG_MAX ) ) FD_TEST( tile->in_link_poll[ i ] ); if( FD_UNLIKELY( i==ctx->failov_in_idx ) ) continue; @@ -490,7 +490,7 @@ poll_set_identity( fd_admin_tile_ctx_t * ctx, fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, tile->id_keyswitch_obj_id ); memcpy( sign->bytes, keypair, 64UL ); if( FD_UNLIKELY( ctx->failover_switch ) ) { - /* Under failover the sign tile selects the junk key or the + /* With failover, the sign tile selects the junk key or the failover identity by its public key. */ sign->param = FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY; fd_memcpy( sign->bytes, keypair+32UL, 32UL ); @@ -683,7 +683,7 @@ failover_identity_check_poll( fd_admin_tile_ctx_t * ctx ) { return result; } -/* Under failover set-identity still switches the same way. */ +/* With failover, set-identity still switches the same way. */ static void failover_operator_begin( fd_admin_tile_ctx_t * ctx, @@ -775,7 +775,7 @@ set_identity( fd_admin_tile_ctx_t * ctx, return; } - /* Under failover a set-identity to the key already installed does + /* With failover, a set-identity to the key already installed does nothing, so failover is not restarted. */ if( FD_UNLIKELY( ctx->failover_enabled && fd_memeq( req->keypair+32UL, ctx->identity_pubkey, 32UL ) ) ) { FD_LOG_NOTICE(( "set-identity: `%s` is already the identity, nothing to do", old_identity )); diff --git a/src/discof/admin/test_admin_tile.c b/src/discof/admin/test_admin_tile.c index e330f2d83af..698a041f6cd 100644 --- a/src/discof/admin/test_admin_tile.c +++ b/src/discof/admin/test_admin_tile.c @@ -195,7 +195,7 @@ test_set_identity( void ) { static uchar adminctl_mem[ 1UL<<18 ] __attribute__((aligned(FD_ADMINCTL_ALIGN))); -/* Under failover, set-identity changes the failover identity unless the +/* With failover, set-identity changes the failover identity unless the key is our junk key. OPERATOR waits while the link has no room and failover commands are busy until it goes out. A switch request from before set-identity gets STALE with the failover identity. */ diff --git a/src/discof/failover/fd_failover_channel.c b/src/discof/failover/fd_failover_channel.c index dd1d17c178a..75c7c16c2e3 100644 --- a/src/discof/failover/fd_failover_channel.c +++ b/src/discof/failover/fd_failover_channel.c @@ -607,7 +607,7 @@ hello_err_name( int err ) { switch( err ) { case HELLO_ERR_TLS_KEY: return "its junk key is not the key TLS authenticated, check who can reach the failover port"; case FD_FAILOVER_HELLO_ERR_VERSION: return "the protocol version differs, run the same Firedancer version on both machines"; - case FD_FAILOVER_HELLO_ERR_STAKED: return "the failover identity differs, check that both machines run [paths.identity_key] with the same keypair, or the same key from their last `set-identity`"; + case FD_FAILOVER_HELLO_ERR_STAKED: return "the failover identity differs, both machines need the same handoff target, from [paths.identity_key] or their last `set-identity`"; case FD_FAILOVER_HELLO_ERR_VOTE_ACCT: return "the vote account differs, check that [paths.vote_account] is the same on both machines"; case FD_FAILOVER_HELLO_ERR_JUNK_EQ: return "its junk key is ours, check that the address is the other validator"; case FD_FAILOVER_HELLO_ERR_JUNK_STAKE: return "a boot identity is the staked key, the standby must use a different identity"; diff --git a/src/discof/failover/fd_failover_tile.c b/src/discof/failover/fd_failover_tile.c index e64a5c5dc17..eeabdaba785 100644 --- a/src/discof/failover/fd_failover_tile.c +++ b/src/discof/failover/fd_failover_tile.c @@ -415,10 +415,9 @@ privileged_init( fd_topo_t const * topo, char staked_b58[ FD_BASE58_ENCODED_32_SZ ]; fd_base58_encode_32( ctx->hello.junk_pubkey, NULL, junk_b58 ); fd_base58_encode_32( ctx->hello.staked_pubkey, NULL, staked_b58 ); - if( ctx->client_only ) FD_LOG_NOTICE(( "failover listener is off, `failover promote ` can request an outgoing handoff while this machine retains its configured identity `%s`", junk_b58 )); - else FD_LOG_NOTICE(( "failover is on, booting as a standby under junk identity `%s` for staked identity `%s`, listening on `" FD_IP4_ADDR_FMT ":%u`, " - "this machine does not vote until `failover promote` hands it the identity from the active, or `failover promote --force` takes it when no machine holds it, " - "the junk identity is this machine's own unique key, never copy it", junk_b58, staked_b58, FD_IP4_ADDR_FMT_ARGS( tile->failov.listen_addr ), (uint)tile->failov.port )); + if( !ctx->client_only ) FD_LOG_NOTICE(( "failover is on, booting as a standby under junk identity `%s` for staked identity `%s`, listening on `" FD_IP4_ADDR_FMT ":%u`, " + "this machine does not vote until `failover promote` hands it the identity from the active, or `failover promote --force` takes it when no machine holds it", + junk_b58, staked_b58, FD_IP4_ADDR_FMT_ARGS( tile->failov.listen_addr ), (uint)tile->failov.port )); FD_TEST( FD_SCRATCH_ALLOC_FINI( l, scratch_align() )==(ulong)scratch+scratch_footprint( tile ) ); } From 430a4e30da467a65389facf40d58556cc27776fd Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Mon, 26 Oct 2026 19:33:47 +0000 Subject: [PATCH 20/23] failover: keep validators that do not use failover unchanged Every validator with a vote account gets the failover tile and links so it can request a handoff, so failover code ran on every voter. A voter that never used failover set the vote bound from its restart floor, marked its dropped votes unsent and applied the failover leader floor on set-identity, so it voted differently from a node without failover. Rename normal_startup to failover_engaged, set at boot on a standby and at the first adoption on a client. The vote bound from the restart floor, unsent marks, the leader floor, the failover logs and the history export now apply only once it is set. Until then votor publishes only its replay slots. The failov tile also took a [layout.affinity] entry and moved every tile created after it to the next CPU, so a manual affinity string stopped working. It now floats and comes after every other tile, like the waker, and the affinity count leaves it out. A standby now dials the active on port 8010 unless `failover promote --port` gives another, instead of assuming the active listens on its own [failover.listen_port]. Co-Authored-By: Claude Opus 5.5 --- src/app/firedancer/commands/failover.c | 5 +- src/app/firedancer/config/default.toml | 3 +- src/app/firedancer/topology.c | 43 +++++++-------- src/discof/failover/fd_failover_proto.h | 4 ++ src/discof/failover/fd_failover_tile.c | 12 ++--- src/discof/failover/test_failover_tile.c | 1 - src/discof/votor/fd_votor_tile.c | 67 +++++++++++++++--------- src/discof/votor/test_votor_tile.c | 9 ++-- 8 files changed, 80 insertions(+), 64 deletions(-) diff --git a/src/app/firedancer/commands/failover.c b/src/app/firedancer/commands/failover.c index acc761554c0..cf9c0e6f24f 100644 --- a/src/app/firedancer/commands/failover.c +++ b/src/app/firedancer/commands/failover.c @@ -427,9 +427,8 @@ failover_args_help( fd_action_help_t * help ) { fd_action_help_arg( help, "--address", "
", "Only without --force. The active to ask, an IPv4\n" "address or host name, instead of the address gossip shows for it,\n" "for example when the active listens on a private network" ); - fd_action_help_arg( help, "--port", "", "Only without --force. The active's failover port.\n" - "Without it we dial our own [failover.listen_port], so both machines\n" - "normally use the same port" ); + fd_action_help_arg( help, "--port", "", "Only without --force. The active's failover port, 8010 by\n" + "default" ); } static void diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index e382e882053..653e5a5f099 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -2004,7 +2004,8 @@ telemetry = true # the listener to a private network. listen_address = "0.0.0.0" - # The port to listen on for handoff requests. + # The port to listen on for handoff requests. The other validator + # dials 8010 unless `failover promote --port` gives another port. listen_port = 8010 # These options can be useful for development, but should not be used diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index 54796b0d583..976e7add201 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -560,7 +560,7 @@ fd_topo_initialize( config_t * config ) { FOR(sign_tile_cnt-1) fd_topob_link( topo, "sign_repair", "sign_repair", 256UL, sizeof(fd_ed25519_sig_t), 1UL ); if( FD_UNLIKELY( failover_enabled ) ) { - /**/ fd_topob_link( topo, "failov_sign", "failov_sign", 128UL, 130UL, 1UL ); /* member certificate or TLS 1.3 CertificateVerify payload */ + /**/ fd_topob_link( topo, "failov_sign", "failov_sign", 128UL, 130UL, 1UL ); /**/ fd_topob_link( topo, "sign_failov", "sign_failov", 128UL, sizeof(fd_ed25519_sig_t), 1UL ); } @@ -667,7 +667,6 @@ fd_topo_initialize( config_t * config ) { } fd_topo_tile_t * admin_tile = fd_topob_tile( topo, "admin", "admin", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 ); - if( FD_UNLIKELY( failover_enabled ) ) fd_topob_tile( topo, "failov", "failov", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 ); /* topo, tile_name, tile_kind_id, fseq_wksp, link_name, link_kind_id, reliable, polled */ FOR(gossvf_tile_cnt) for( ulong j=0UL; jtiles.gui.enabled ) { fd_topob_tile_out( topo, "gossip", 0UL, "gossip_ciseen", 0UL ); } - /* Failover cannot backpressure validator work. An input overrun - disables failover until restart instead of trusting stale state. */ - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "gossip_ciseen", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /* contact infos for the peer's address */ /**/ fd_topob_tile_out( topo, "admin", 0UL, "admin_failov", 0UL ); - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "admin_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); - /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_admin", 0UL ); /**/ fd_topob_tile_in ( topo, "admin", 0UL, "metric_in", "failov_admin", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); } - if( FD_UNLIKELY( failover_enabled ) ) { - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_hist", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); - /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_votor", 0UL ); - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); - } FOR(execrp_tile_cnt) fd_topob_tile_in ( topo, "execrp", i, "metric_in", "replay_execrp", i, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); FOR(execrp_tile_cnt) fd_topob_tile_out( topo, "execrp", i, "execrp_replay", i ); @@ -946,8 +935,6 @@ fd_topo_initialize( config_t * config ) { if( FD_UNLIKELY( failover_enabled ) ) { /**/ fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "failov_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); - /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_sign", 0UL ); - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "sign_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED ); /**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_failov", 0UL ); } @@ -1102,17 +1089,31 @@ fd_topo_initialize( config_t * config ) { } } + if( FD_UNLIKELY( failover_enabled ) ) { + /**/ fd_topob_tile( topo, "failov", "failov", "metric_in", ULONG_MAX, 0, 0, 0 ); + + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "gossip_ciseen", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "admin_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_admin", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_hist", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_votor", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_sign", 0UL ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "sign_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED ); + } + ulong pinned_tile_cnt = topo->tile_cnt - (ulong)!!failover_enabled; + if( FD_LIKELY( !is_auto_affinity ) ) { - if( FD_UNLIKELY( affinity_tile_cnttile_cnt ) ) + if( FD_UNLIKELY( affinity_tile_cnttile_cnt, affinity_tile_cnt )); - if( FD_UNLIKELY( affinity_tile_cnt>topo->tile_cnt ) ) + pinned_tile_cnt, affinity_tile_cnt )); + if( FD_UNLIKELY( affinity_tile_cnt>pinned_tile_cnt ) ) FD_LOG_WARNING(( "The topology you are using has %lu tiles, but the CPU affinity specified in the config tile as [layout.affinity] provides for %lu cores. " "Not all cores in the affinity will be used by Firedancer. You may wish to increase the number of tiles in the system by increasing " "individual tile counts in the [layout] section of the configuration file.", - topo->tile_cnt, affinity_tile_cnt )); + pinned_tile_cnt, affinity_tile_cnt )); } else { ushort blocklist_cores[ FD_TILE_MAX ]; topo->blocklist_cores_cnt = fd_topob_parse_affinity_cstr( config->layout.blocklist_cores, blocklist_cores, 0, 0 ); @@ -1161,8 +1162,6 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_tile_in ( topo, "rpc", 0UL, "metric_in", "votor_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); } - /* failov_votor has to come before the unpolled sign_votor, the votor looks - up in_kind by polled input index. */ if( FD_UNLIKELY( failover_enabled ) ) { /**/ fd_topob_tile_in ( topo, "votor", 0UL, "metric_in", "failov_votor", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "votor", 0UL, "votor_hist", 0UL ); @@ -1434,10 +1433,6 @@ wfs_decode_bank_hash( fd_config_t const * config, void fd_topo_configure_tile( fd_topo_tile_t * tile, fd_config_t * config ) { - /* Failover machines boot under the junk identity, only a promotion or - set-identity installs a voting key. A demoted machine rebooting under the - staked key would make the doppelganger rule shut down the promoted - peer instead. */ char const * identity_key_path = config->firedancer.failover.enabled && fd_topo_find_tile( &config->topo, "failov", 0UL )!=ULONG_MAX ? FD_KEYLOAD_FAILOVER_JUNK : config->paths.identity_key; diff --git a/src/discof/failover/fd_failover_proto.h b/src/discof/failover/fd_failover_proto.h index c9849c89dea..9921df7e44e 100644 --- a/src/discof/failover/fd_failover_proto.h +++ b/src/discof/failover/fd_failover_proto.h @@ -10,6 +10,10 @@ /* Protocol version */ #define FD_FAILOVER_VERSION (1U) +/* The port a standby dials unless `failover promote --port` gives + another. It is also the default [failover.listen_port]. */ +#define FD_FAILOVER_DEFAULT_PORT ((ushort)8010) + /* Message types */ #define FD_FAILOVER_MSG_HELLO (0U) #define FD_FAILOVER_MSG_HANDOFF_REQUEST (1U) diff --git a/src/discof/failover/fd_failover_tile.c b/src/discof/failover/fd_failover_tile.c index eeabdaba785..4e4385117e2 100644 --- a/src/discof/failover/fd_failover_tile.c +++ b/src/discof/failover/fd_failover_tile.c @@ -197,8 +197,7 @@ struct fd_failover_tile_ctx { fd_failover_hello_t hello; /* the HELLO we send */ ulong role; /* FD_FAILOVER_ROLE_* */ uint cmd_addr; /* from `failover promote --address`, 0 when gossip finds the active */ - ushort cmd_port; /* from `failover promote --port`, 0 for our own port */ - ushort port; /* our failover listen port */ + ushort cmd_port; /* from `failover promote --port`, 0 for the default port */ fd_failover_channel_t * channel; /* the TLS channel to the peer */ fd_failover_session_t session; /* the session we last saw on the channel */ long poll_at; /* next idle socket poll */ @@ -392,7 +391,6 @@ privileged_init( fd_topo_t const * topo, fd_hex_decode( ctx->hello.commit, fd_commit_ref_cstr, sizeof(ctx->hello.commit) )!=sizeof(ctx->hello.commit) ) ) { fd_memset( ctx->hello.commit, 0, sizeof(ctx->hello.commit) ); } - ctx->port = tile->failov.port; ctx->own_gossip = tile->failov.gossip_addr; if( FD_UNLIKELY( tile->failov.gossip_host[ 0 ]!='\0' ) ) { @@ -537,10 +535,10 @@ sync_session( fd_failover_tile_ctx_t * ctx ) { } /* Returns the port a handoff dials, from `failover promote --port` if - given, else our own. */ + given, else the default port. */ static inline ushort dial_port( fd_failover_tile_ctx_t const * ctx ) { - return ctx->cmd_port ? ctx->cmd_port : ctx->port; + return ctx->cmd_port ? ctx->cmd_port : FD_FAILOVER_DEFAULT_PORT; } static void @@ -1942,7 +1940,7 @@ preconnect( fd_failover_tile_ctx_t * ctx, now-ctx->staked_seen_atchannel, 0U, 0 ); else if( fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ) - fd_failover_channel_init_dialer( ctx->channel, ctx->staked_addr, ctx->port ); + fd_failover_channel_init_dialer( ctx->channel, ctx->staked_addr, FD_FAILOVER_DEFAULT_PORT ); } /* An operator command is required before sending a handoff request. */ @@ -2217,7 +2215,7 @@ status_snapshot( fd_failover_tile_ctx_t const * ctx, resp->peer_boot_id = ctx->session.peer_boot_id; resp->peer_addr = ctx->request.id ? ctx->request.addr : ctx->staked_addr; resp->peer_addr_cmd = (uchar)( ctx->request.id && ctx->cmd_addr ); - resp->peer_port = ctx->request.id ? dial_port( ctx ) : ctx->port; + resp->peer_port = ctx->request.id ? dial_port( ctx ) : FD_FAILOVER_DEFAULT_PORT; resp->handoff_id = ctx->last_requested ? ctx->request.last_id : ctx->handoff.id; resp->handoff_result = (uchar)( ctx->last_requested ? ctx->request.result : ctx->handoff.result ); resp->promote_source = (uchar)promote_source( ctx ); diff --git a/src/discof/failover/test_failover_tile.c b/src/discof/failover/test_failover_tile.c index 1d31e46e4bb..9eba738175f 100644 --- a/src/discof/failover/test_failover_tile.c +++ b/src/discof/failover/test_failover_tile.c @@ -464,7 +464,6 @@ operator_cmd( fd_failover_tile_ctx_t * ctx, static void test_request_address( void ) { fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); - ctx->port = 8010; ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); ctx->staked_addr = FD_IP4_ADDR(10,0,0,3); diff --git a/src/discof/votor/fd_votor_tile.c b/src/discof/votor/fd_votor_tile.c index 84e663ba0e8..3dc4d9d13fa 100644 --- a/src/discof/votor/fd_votor_tile.c +++ b/src/discof/votor/fd_votor_tile.c @@ -396,7 +396,7 @@ struct fd_votor_tile { /* Failover, on when the failover links are in the topology */ int failover_enabled; - int normal_startup; /* ordinary voting until the first failover adoption */ + int failover_engaged; /* failover voting rules apply, from boot on a standby, from the first adoption on a client */ fd_pubkey_t boot_id_key; /* the identity we booted with, the junk key under failover */ int failover_hist_adopted; /* a history, possibly empty, was adopted since the last switch */ int vote_authority; /* our rank may be installed, 0 keeps us unranked */ @@ -1191,7 +1191,7 @@ update_own_bls_key( fd_votor_tile_t * ctx ) { for( ulong rank=0UL; rankvalidator_cnt; rank++ ) { if( FD_UNLIKELY( !memcmp( epoch_info->validators[ rank ].id_key, ctx->id_key.uc, sizeof(ag_id_key_t) ) ) ) own_rank = (ushort)rank; } - if( FD_UNLIKELY( ctx->failover_enabled && !ctx->vote_authority ) ) own_rank = USHORT_MAX; /* under failover we stay unranked while we may not vote */ + if( FD_UNLIKELY( ctx->failover_enabled && !ctx->vote_authority ) ) own_rank = USHORT_MAX; /* with failover we stay unranked while we may not vote */ uchar const * own_bls_key = NULL; /* NULL if we are not ranked or cannot sign with our key, in which case we do not vote */ if( FD_LIKELY( own_rank!=USHORT_MAX ) ) { ag_validator_info_t const * validator = &epoch_info->validators[ own_rank ]; @@ -1383,12 +1383,23 @@ publish_hist( fd_votor_tile_t * ctx, msg->replay_slot = ctx->highest_completed_slot ? ctx->highest_completed_slot : ULONG_MAX; msg->root_slot = ctx->root_slot; msg->has_vote = has_vote; - /* Preserve the identity switch floor for the next adopter. */ - if( FD_UNLIKELY( ctx->vote_history_slot ) ) ag_votor_set_vote_bound( ctx->votor, ctx->vote_history_slot-1UL ); - msg->truncated = ag_votor_hist_export( ctx->votor, leader_floor( ctx ), &msg->hist ); + if( FD_LIKELY( !ctx->failover_engaged ) ) { + /* Until failover engages we hand off nothing and the failover tile + only reads our replay slots, so we skip the export and keep the + plain voting rules. */ + msg->hist.anchor = 0UL; + msg->hist.last_leader_slot = ULONG_MAX; + msg->hist.vote_bound = ULONG_MAX; + msg->hist.rec_cnt = 0UL; + msg->truncated = 0; + } else { + /* Preserve the identity switch floor for the next adopter. */ + if( FD_UNLIKELY( ctx->vote_history_slot ) ) ag_votor_set_vote_bound( ctx->votor, ctx->vote_history_slot-1UL ); + msg->truncated = ag_votor_hist_export( ctx->votor, leader_floor( ctx ), &msg->hist ); + if( FD_UNLIKELY( msg->truncated && !ctx->hist_was_truncated ) ) FD_LOG_WARNING(( "vote history export dropped windows below slot %lu, finality is far behind our votes", ag_hist_first_slot( msg->hist.anchor ) )); + ctx->hist_was_truncated = msg->truncated; + } msg->vote_slot = ag_hist_tip( &msg->hist ); /* the failover tile checks the history ends here */ - if( FD_UNLIKELY( msg->truncated && !ctx->hist_was_truncated ) ) FD_LOG_WARNING(( "vote history export dropped windows below slot %lu, finality is far behind our votes", ag_hist_first_slot( msg->hist.anchor ) )); - ctx->hist_was_truncated = msg->truncated; ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); fd_stem_publish( stem, ctx->hist_out_idx, FD_VOTOR_HIST_SIG, ctx->hist_out_chunk, sizeof(fd_votor_hist_msg_t), 0UL, tspub, tspub ); ctx->hist_out_chunk = fd_dcache_compact_next( ctx->hist_out_chunk, sizeof(fd_votor_hist_msg_t), ctx->hist_out_chunk0, ctx->hist_out_wmark ); @@ -1455,7 +1466,7 @@ failover_adopt_hist( fd_votor_tile_t * ctx, /* Reject a history older than the votes we sent as the staked identity. */ result.result = FD_VOTOR_ADOPT_ERR_STALE; - if( FD_UNLIKELY( !ctx->normal_startup && ctx->last_vote_slot!=ULONG_MAX && tiplast_vote_slot ) ) { + if( FD_UNLIKELY( ctx->failover_engaged && ctx->last_vote_slot!=ULONG_MAX && tiplast_vote_slot ) ) { FD_LOG_WARNING(( "failover: refusing a vote history through slot %lu, this machine already voted as this identity through slot %lu", tip, ctx->last_vote_slot )); return result; } @@ -1489,22 +1500,25 @@ failover_adopt_hist( fd_votor_tile_t * ctx, /* Applies the failover voting rule at the end of an identity switch. */ static void failover_switched( fd_votor_tile_t * ctx ) { - /* Skip leader windows this identity may already have led. */ + /* Skip leader windows this identity may already have led. Until + failover engages, set-identity keeps its plain rules. */ ulong floor = leader_floor( ctx ); - if( FD_UNLIKELY( floor!=ULONG_MAX ) ) { + if( FD_UNLIKELY( ctx->failover_engaged && floor!=ULONG_MAX ) ) { ulong next_window = ag_first_slot_in_window( ctx->highest_parent_ready_slot )+AG_SLOTS_PER_WINDOW; ulong floor_window = ag_first_slot_in_window( floor )+AG_SLOTS_PER_WINDOW; if( FD_UNLIKELY( floor_window>next_window ) ) { ctx->next_leader_slot = fd_multi_epoch_leaders_get_next_slot( ctx->mleaders, fd_ulong_max( fd_ulong_max( ctx->curr_epoch_slot, floor_window ), ctx->vote_history_slot ), &ctx->id_key ); } } - FD_BASE58_ENCODE_32_BYTES( ctx->id_key.uc, id_key_b58 ); int operator = !!FD_VOLATILE_CONST( ctx->id_keyswitch->operator ); int standby = fd_pubkey_eq( &ctx->id_key, &ctx->boot_id_key ); - ctx->vote_authority = ( ctx->normal_startup && operator ) || ( !standby && ( operator || ctx->failover_hist_adopted ) ); - FD_LOG_NOTICE(( "failover: switched to identity `%s`%s, this machine is now %s", id_key_b58, operator ? " by set-identity" : "", - ctx->vote_authority ? "the active voter" : "a standby" )); - if( FD_UNLIKELY( !operator && !standby && !ctx->failover_hist_adopted ) ) FD_LOG_WARNING(( "failover: the staked identity was installed without an adopted vote history, refusing to vote until an adoption precedes an identity switch" )); + ctx->vote_authority = ( !ctx->failover_engaged && operator ) || ( !standby && ( operator || ctx->failover_hist_adopted ) ); + if( FD_UNLIKELY( ctx->failover_engaged ) ) { + FD_BASE58_ENCODE_32_BYTES( ctx->id_key.uc, id_key_b58 ); + FD_LOG_NOTICE(( "failover: switched to identity `%s`%s, this machine is now %s", id_key_b58, operator ? " by set-identity" : "", + ctx->vote_authority ? "the active voter" : "a standby" )); + if( FD_UNLIKELY( !operator && !standby && !ctx->failover_hist_adopted ) ) FD_LOG_WARNING(( "failover: the staked identity was installed without an adopted vote history, refusing to vote until an adoption precedes an identity switch" )); + } ctx->failover_hist_adopted = 0; ctx->id_keyswitch->result = ctx->hist_seq; } @@ -1517,23 +1531,28 @@ failover_slot_completed( fd_votor_tile_t * ctx, if( FD_LIKELY( slot_completed->root_slot!=ULONG_MAX ) ) ctx->root_slot = slot_completed->root_slot; } -/* Records a vote we cast and publishes the history, marking the vote - unsent if it never left the machine. */ +/* Records a vote we cast. Once failover engages it also marks a vote + that never left the machine unsent and publishes the history. Until + then the plain voting rules apply and nothing is published here, our + replay slots go out as replay completes. */ static void failover_after_vote( fd_votor_tile_t * ctx, fd_stem_context_t * stem, ulong vote_slot, int sent ) { if( FD_LIKELY( sent && ( ctx->last_vote_slot==ULONG_MAX || vote_slot>ctx->last_vote_slot ) ) ) ctx->last_vote_slot = vote_slot; + if( FD_LIKELY( !ctx->failover_engaged ) ) return; if( FD_UNLIKELY( !sent ) ) ag_votor_mark_unsent( ctx->votor, &ctx->scratch.vote ); publish_hist( ctx, stem, sent ); } -/* Records our LEADER and publishes the history. */ +/* Records our LEADER. Once failover engages it also publishes the + history. */ static void failover_after_leader( fd_votor_tile_t * ctx, fd_stem_context_t * stem ) { ctx->last_leader_slot = ctx->next_leader_slot; + if( FD_LIKELY( !ctx->failover_engaged ) ) return; publish_hist( ctx, stem, 0 ); } @@ -1567,7 +1586,7 @@ failover_init( fd_votor_tile_t * ctx, /* A configured standby waits for a switch. An outgoing client keeps ordinary voting until its first successful adoption. */ ctx->boot_id_key = ctx->id_key; - ctx->normal_startup = !tile->votor.failover_standby; + ctx->failover_engaged = !!tile->votor.failover_standby; ctx->vote_authority = !tile->votor.failover_standby; ctx->failover_hist_adopted = 0; ctx->hist_was_truncated = 0; @@ -1597,7 +1616,7 @@ failover_adopt_answer( fd_votor_tile_t * ctx, ulong sig, fd_stem_context_t * stem ) { fd_votor_adopt_result_t result; - if( FD_UNLIKELY( !fd_pubkey_eq( &ctx->id_key, &ctx->boot_id_key ) || ( ctx->vote_authority && !ctx->normal_startup ) ) ) { + if( FD_UNLIKELY( !fd_pubkey_eq( &ctx->id_key, &ctx->boot_id_key ) || ( ctx->vote_authority && ctx->failover_engaged ) ) ) { FD_LOG_WARNING(( "failover: refusing a vote history adoption outside standby or client startup" )); result = (fd_votor_adopt_result_t){ .result=FD_VOTOR_ADOPT_ERR_INVALID, .root=ULONG_MAX, .vote_slot=ULONG_MAX, .vote_bound=ULONG_MAX }; } else { @@ -1605,7 +1624,7 @@ failover_adopt_answer( fd_votor_tile_t * ctx, their whole windows so the target cannot finalize them. Export the bound with future histories so another machine preserves it. */ ulong client_bound = ULONG_MAX; - if( FD_UNLIKELY( ctx->normal_startup ) ) { + if( FD_UNLIKELY( !ctx->failover_engaged ) ) { ag_hist_t local; ag_votor_hist_export( ctx->votor, ULONG_MAX, &local ); ulong tip = ag_hist_tip( &local ); @@ -1613,7 +1632,7 @@ failover_adopt_answer( fd_votor_tile_t * ctx, } result = failover_adopt_hist( ctx, ctx->adopt_req, ctx->adopt_req_sz ); if( FD_LIKELY( result.result==FD_VOTOR_ADOPT_SUCCESS ) ) { - if( FD_UNLIKELY( ctx->normal_startup ) ) { + if( FD_UNLIKELY( !ctx->failover_engaged ) ) { if( client_bound!=ULONG_MAX ) { ag_votor_set_vote_bound( ctx->votor, client_bound ); raise_leader_floor( ctx, client_bound ); @@ -1621,7 +1640,7 @@ failover_adopt_answer( fd_votor_tile_t * ctx, } ctx->last_vote_slot = ULONG_MAX; } - ctx->normal_startup = 0; + ctx->failover_engaged = 1; ctx->vote_authority = 0; update_own_rank( ctx ); update_own_bls_key( ctx ); @@ -2697,7 +2716,7 @@ before_frag( fd_votor_tile_t * ctx, case IN_KIND_FAILOV: /* Drain votes and pool events produced under the boot identity before changing the history or removing its rank. */ - if( FD_UNLIKELY( ctx->normal_startup && ctx->init && + if( FD_UNLIKELY( !ctx->failover_engaged && ctx->init && ( ag_votor_metrics( ctx->votor ).vote_events_cnt || ag_pool_metrics( ctx->pool ).pool_events_cnt ) ) ) return -1; return 0; default: diff --git a/src/discof/votor/test_votor_tile.c b/src/discof/votor/test_votor_tile.c index 9dd9fd47625..a81b2de467a 100644 --- a/src/discof/votor/test_votor_tile.c +++ b/src/discof/votor/test_votor_tile.c @@ -1756,6 +1756,7 @@ hist_setup( fd_votor_tile_t * ctx ) { ctx->boot_id_key = ctx->id_key; ctx->curr_epoch_info = epoch_info; ctx->failover_enabled = 1; + ctx->failover_engaged = 1; ctx->hist_out_idx = ULONG_MAX; ctx->failov_out_idx = ULONG_MAX; ctx->last_leader_slot = ULONG_MAX; @@ -2806,7 +2807,7 @@ test_failover_off( void ) { hist_teardown( &ctx ); } -/* Test that under failover the staked key votes only after an adoption or +/* Test that with failover the staked key votes only after an adoption or set-identity, and the junk key never does. */ static void @@ -2921,7 +2922,7 @@ test_adopt_refused_while_voting( void ) { FD_TEST( ag_votor_vote_bound( ctx.votor )==ULONG_MAX ); /* A client retains its old authority if the imported slots cannot fit. */ - ctx.normal_startup = 1; + ctx.failover_engaged = 0; ag_hist_t hist = { .anchor = 12UL, .last_leader_slot = 80UL, .vote_bound = 83UL, .rec_cnt = AG_HIST_MAX }; for( ulong i=0UL; iresult==FD_VOTOR_ADOPT_ERR_CAPACITY && !ctx.failover_hist_adopted ); - FD_TEST( ctx.normal_startup && ctx.vote_authority ); + FD_TEST( !ctx.failover_engaged && ctx.vote_authority ); FD_TEST( ctx.adopted_last_leader_slot==ULONG_MAX ); FD_TEST( ag_votor_vote_bound( ctx.votor )==ULONG_MAX ); FD_TEST( ag_votor_metrics( ctx.votor ).highest_final_cert_slot==0UL ); FD_TEST( ag_votor_metrics( ctx.votor ).slot_state_pool_free==free ); - ctx.normal_startup = 0; + ctx.failover_engaged = 1; ctx.vote_authority = 0; FD_STORE( ulong, ctx.adopt_req, 9UL ); ctx.adopt_req_sz = FD_VOTOR_ADOPT_EMPTY_SZ; From e056656e018302f6cb5ca8c0c0b623abc22f02be Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Mon, 26 Oct 2026 19:33:47 +0000 Subject: [PATCH 21/23] votor: fence the boot identity's windows on every adoption A standby records its junk identity's votes in its slot states without sending them. Adoption only writes the slots the peer's history covers, so after a handoff the staked identity could send a final vote for, or notarize on top of, a slot it never voted notar for. Clients already fenced the windows of their own boot identity's votes. Apply the same whole-window bound on every adoption, including a standby and an empty forced history. Co-Authored-By: Claude Opus 5.5 --- src/discof/votor/fd_votor_tile.c | 32 +++++++++++++++----------------- 1 file changed, 15 insertions(+), 17 deletions(-) diff --git a/src/discof/votor/fd_votor_tile.c b/src/discof/votor/fd_votor_tile.c index 3dc4d9d13fa..37614ef8acc 100644 --- a/src/discof/votor/fd_votor_tile.c +++ b/src/discof/votor/fd_votor_tile.c @@ -1620,26 +1620,24 @@ failover_adopt_answer( fd_votor_tile_t * ctx, FD_LOG_WARNING(( "failover: refusing a vote history adoption outside standby or client startup" )); result = (fd_votor_adopt_result_t){ .result=FD_VOTOR_ADOPT_ERR_INVALID, .root=ULONG_MAX, .vote_slot=ULONG_MAX, .vote_bound=ULONG_MAX }; } else { - /* The client's old notar votes belong to its boot identity. Fence - their whole windows so the target cannot finalize them. Export - the bound with future histories so another machine preserves it. */ - ulong client_bound = ULONG_MAX; - if( FD_UNLIKELY( !ctx->failover_engaged ) ) { - ag_hist_t local; - ag_votor_hist_export( ctx->votor, ULONG_MAX, &local ); - ulong tip = ag_hist_tip( &local ); - if( tip!=ULONG_MAX ) client_bound = ag_first_slot_in_window( tip )+AG_SLOTS_PER_WINDOW-1UL; - } + /* Our slot states hold the votes of the boot identity, a standby's + junk key or a client's own key, and none of them went out as the + target. Fence their whole windows so the target cannot finalize + them or build on them. Export the bound with future histories so + another machine preserves it. */ + ulong boot_bound = ULONG_MAX; + ag_hist_t local; + ag_votor_hist_export( ctx->votor, ULONG_MAX, &local ); + ulong tip = ag_hist_tip( &local ); + if( tip!=ULONG_MAX ) boot_bound = ag_first_slot_in_window( tip )+AG_SLOTS_PER_WINDOW-1UL; result = failover_adopt_hist( ctx, ctx->adopt_req, ctx->adopt_req_sz ); if( FD_LIKELY( result.result==FD_VOTOR_ADOPT_SUCCESS ) ) { - if( FD_UNLIKELY( !ctx->failover_engaged ) ) { - if( client_bound!=ULONG_MAX ) { - ag_votor_set_vote_bound( ctx->votor, client_bound ); - raise_leader_floor( ctx, client_bound ); - result.vote_bound = ag_votor_vote_bound( ctx->votor ); - } - ctx->last_vote_slot = ULONG_MAX; + if( boot_bound!=ULONG_MAX ) { + ag_votor_set_vote_bound( ctx->votor, boot_bound ); + raise_leader_floor( ctx, boot_bound ); + result.vote_bound = ag_votor_vote_bound( ctx->votor ); } + if( FD_UNLIKELY( !ctx->failover_engaged ) ) ctx->last_vote_slot = ULONG_MAX; ctx->failover_engaged = 1; ctx->vote_authority = 0; update_own_rank( ctx ); From e1f2379f6a719dd7da2ad186aa634949ff2f0551 Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Tue, 27 Oct 2026 15:56:38 +0000 Subject: [PATCH 22/23] failover: boot under [paths.identity_key] and take the target from failover promote Co-Authored-By: Claude Fable 5.1 --- book/api/cli.md | 92 +- book/snippets/commands/failover-promote.ansi | 8 +- book/snippets/commands/failover-status.ansi | 8 +- src/app/firedancer/commands/failover.c | 352 ++--- src/app/firedancer/commands/set_identity.c | 8 +- src/app/firedancer/config/default.toml | 20 +- src/app/firedancer/topology.c | 84 +- src/app/shared/commands/run/run.c | 33 +- src/app/shared/commands/run/run1.c | 5 +- src/app/shared/fd_config.c | 9 - src/app/shared/fd_config_json.c | 4 +- src/app/shared/test_config_json.c | 8 +- src/app/shared/test_config_parse.c | 2 +- src/choreo/votor/ag_votor.c | 8 + src/disco/events/generated/fd_event_gen.h | 2 +- src/disco/events/schema/admin_command.json | 2 +- src/disco/events/schema/events.proto | 2 +- src/disco/keyguard/fd_keyload.c | 45 +- src/disco/keyguard/fd_keyload.h | 8 +- src/disco/keyguard/fd_keyswitch.h | 2 +- src/disco/keyguard/fd_sign_tile.c | 97 +- src/disco/keyguard/test_sign_tile.c | 32 +- src/disco/topo/fd_topo.c | 10 +- src/disco/topo/fd_topo.h | 9 +- src/disco/topo/fd_topo_run.c | 2 +- src/discof/admin/fd_admin_tile.c | 131 +- src/discof/admin/fd_adminctl.h | 89 +- src/discof/admin/test_admin_tile.c | 33 +- src/discof/failover/fd_failover_bus.h | 17 +- src/discof/failover/fd_failover_channel.c | 72 +- src/discof/failover/fd_failover_channel.h | 7 +- src/discof/failover/fd_failover_proto.c | 25 +- src/discof/failover/fd_failover_proto.h | 32 +- src/discof/failover/fd_failover_tile.c | 1287 ++++++++++-------- src/discof/failover/fd_failover_wire.c | 41 +- src/discof/failover/fd_failover_wire.h | 38 +- src/discof/failover/test_failover_channel.c | 2 +- src/discof/failover/test_failover_proto.c | 34 +- src/discof/failover/test_failover_tile.c | 305 +++-- src/discof/failover/test_failover_tile_ag.c | 78 +- src/discof/failover/test_failover_wire.c | 26 +- src/discof/gossip/fd_gossvf_tile.c | 4 +- src/discof/votor/fd_votor_tile.c | 140 +- src/discof/votor/fd_votor_tile.h | 8 +- src/discof/votor/test_votor_tile.c | 57 +- 45 files changed, 1821 insertions(+), 1457 deletions(-) diff --git a/book/api/cli.md b/book/api/cli.md index 864bbb45dbf..473840d1663 100644 --- a/book/api/cli.md +++ b/book/api/cli.md @@ -344,11 +344,13 @@ than 16). <<< @/snippets/commands/add-authorized-voter.ansi ## `failover promote` -This command asks a remote active validator to hand off its identity and -vote history to the local running standby. Run this command on the -standby. The remote validator must have `[failover.enabled]` set to -`true` to accept the request. Both validators must use the same vote -account. +This command takes an identity key, with its vote history, from the +remote validator currently running it and installs it on this local +validator. Run this command on the machine that should take over, +with the target keypair of the identity to take. The machine that +originally ran it will switch to a random junk key that it generated +at boot. The validators must use the same vote account and have +failover enabled, which is on by default. ```sh firedancer failover promote [] [--name ] [--yes] [--force] [--address
] [--port ] @@ -359,33 +361,37 @@ automatically. If more than one is running, pass `--name ` to select one (see [`ps`](#ps)). It asks for confirmation when run from a terminal unless `--yes` is given. -The local validator finds the active's address through gossip. Use -`--address` to give the remote validator's IPv4 address or host name -instead, for example when connecting over a private network. Use +The local validator finds the address of the machine running the +identity through gossip. Use `--address` to give its IPv4 address or host +name instead, for example when connecting over a private network. Use `--port` if its failover listener does not use the default port 8010. For example, to connect to a known failover listener over a private -network, run the following on the local standby. +network, run the following on the machine that should take over. ```sh -firedancer failover promote --address 192.0.2.10 --port 8010 +firedancer failover promote /path/to/identity.json --address 192.0.2.10 --port 8010 ``` ### Setup -Set `[failover.enabled]` to `true` on both validators. Configure the same -staked identity in `[paths.identity_key]` and the same vote account in -`[paths.vote_account]`. Firedancer generates a fresh junk identity in memory -on each startup. Both validators boot under a junk identity without voting -and listen for handoff requests. +Failover runs on every validator with `[failover.enabled]` set to `true`. +Each validator boots under its own `[paths.identity_key]`, and all must +use the same vote account. The validator that should vote keeps the +identity's keypair there. On every other machine, create a random placeholder +keypair of its own and point `[paths.identity_key]` at it, so it replays +without voting until it takes over. -Once one validator is active, run the following on the standby to request -a handoff. The staked identity keypair is already loaded from the -configuration, so no keypair argument is needed. +```sh +firedancer keys new /home/firedancer/junk.json --config config.toml +``` + +To hand the identity over, run the following on the machine that should +take it, with the keypair of the identity to take. ```sh -firedancer failover promote --name standby -firedancer failover status --name standby +firedancer failover promote /path/to/identity.json +firedancer failover status ``` For example, an accepted request prints the handoff ID and the status @@ -393,16 +399,34 @@ needed to confirm completion. <<< @/snippets/commands/failover-promote.ansi -After the handoff, the old active can request the identity back with -`failover promote`. If both validators are still standbys, first promote -one with `--force` as described below. An optional keypair argument must -match the standby's current failover target. +The machine that gave it up then runs its junk key. To take the identity +back, run `failover promote` again on the demoted machine. + +### Restarts + +A validator always boots under its `[paths.identity_key]`. Before +restarting the machine that gave the identity up, point its +`[paths.identity_key]` at a junk key of its own. If it boots under the +identity while the other machine runs it, the duplicate instance check +stops the other machine. + +Leaving an identity renames its vote history file in +`[paths.vote_history]` to `vote_history-.bin.bak`, and a +validator refuses to boot under a key whose vote history file is missing +while other vote history files are there. + +- To restart a machine that took the identity under its junk key, first + rename `vote_history-.bin.bak` back to `vote_history-.bin`. +- To restart the machine that gave the identity up under a junk key, + first move the `vote_history-*` files out of `[paths.vote_history]`. + The identity's vote history now lives on the other machine. ### Forced promotion -With `--force`, the local standby takes the identity without contacting -the remote validator. This is also needed for first use when both -machines are standbys. +With `--force`, the local validator takes the identity without +contacting the machine running it. Use it when no machine runs the +identity, for example after both restarted, or when the other machine +cannot sign. ::: warning WARNING @@ -415,13 +439,13 @@ Never force promotion on both machines at once. | Arguments | Description | |-----------------------|-------------| -| `` | Optionally provide the target identity keypair file. It must match the standby's current failover target. | +| `` | The keypair file of the identity to take. | `--name ` | Select the validator instance to attach to if more than one is running on this host. | | `--config ` | Optionally use the configuration TOML file the validator was started with. | | `--yes` | Skip the confirmation prompt. | -| `--force` | Take the identity without cooperation from the remote validator, allowing incomplete or empty vote history. | -| `--address
` | Use this IPv4 address or host name for the remote validator instead of the address found through gossip. | -| `--port ` | Set the remote validator's failover TCP port. The default is 8010. | +| `--force` | Forcibly take the identity without cooperation with the machine running it, allowing incomplete or empty vote history. | +| `--address
` | Use this IPv4 address or host name for the machine running the identity instead of the address found through gossip. | +| `--port ` | Set the failover TCP port of the machine running the identity. The default is 8010. | ## `failover status` This command prints the local running validator's failover status. Use @@ -439,9 +463,9 @@ reason to `stderr`. The following example shows a completed handoff. The last handoff ID matches the ID printed by `failover promote` above. The connection closes -after completion, so `peer role` is unknown until another authenticated -session is established. The new standby reconnects when it discovers the -new active through gossip. +after completion, so `peer runs identity` is unknown until another +authenticated session is established. The machine that handed over its +identity reconnects once it is discoverable through gossip. <<< @/snippets/commands/failover-status.ansi diff --git a/book/snippets/commands/failover-promote.ansi b/book/snippets/commands/failover-promote.ansi index 2272ba232fb..f3968e98026 100644 --- a/book/snippets/commands/failover-promote.ansi +++ b/book/snippets/commands/failover-promote.ansi @@ -1,5 +1,5 @@ -$ firedancer failover promote --name standby --yes +$ firedancer failover promote /home/firedancer/identity.json --yes failover promote accepted -role: standby -action: handoff, requesting the active's final vote state -handoff: 123456789, done when `failover status` here shows role: active and last handoff: 123456789, taken +runs identity: no +action: handoff, requesting the final vote state +handoff: 123456789, done when `failover status` here shows runs identity: yes and last handoff: 123456789, taken diff --git a/book/snippets/commands/failover-status.ansi b/book/snippets/commands/failover-status.ansi index ab00e228fa7..f64e53dbb7c 100644 --- a/book/snippets/commands/failover-status.ansi +++ b/book/snippets/commands/failover-status.ansi @@ -1,13 +1,13 @@ -$ firedancer failover status --name standby +$ firedancer failover status failover: enabled -role: active +runs identity: yes action: idle stuck: no link: listening -peer role: unknown, no authenticated failover session +peer runs identity: unknown, no authenticated failover session peer boot id: 0000000000000001 peer address: 192.0.2.10:8010, from gossip last handoff: 123456789, taken -other holder: n/a, this machine holds the identity +other holder: n/a, this machine runs the identity takeover adopts: an empty history (requires --force) coverage floor: slot 300 diff --git a/src/app/firedancer/commands/failover.c b/src/app/firedancer/commands/failover.c index cf9c0e6f24f..ef6e7f654da 100644 --- a/src/app/firedancer/commands/failover.c +++ b/src/app/firedancer/commands/failover.c @@ -6,19 +6,24 @@ #include "../../../disco/topo/fd_dns_resolve.h" #include "../../../util/net/fd_ip4.h" #include "../../../disco/keyguard/fd_keyload.h" +#include "../../../ballet/base58/fd_base58.h" +#include "../../../ballet/ed25519/fd_ed25519.h" #include "../../platform/fd_cap_chk.h" -#include -#include #include #include #include static char const * const ACTION_NAMES[] = { - "idle", "handing off, switching to the junk key", "handing off, waiting for the peer's answer", - "promote, waiting for replay", "promote, waiting for vote history adoption", "promote, switching to the staked key", - "handoff, requesting the active's final vote state", "handoff, waiting for completion", - "handing off, waiting for the vote history stream to drain" + "idle", + "giving up the identity, switching to the junk key", + "giving up the identity, waiting for the peer's answer", + "taking the identity, waiting for replay", + "taking the identity, waiting for vote history adoption", + "taking the identity, switching to it", + "taking the identity, requesting the final vote state", + "taking the identity, waiting for the peer to record the handoff", + "giving up the identity, waiting for the vote history stream to drain" }; static char const * const SESSION_NAMES[] = { "listening", "dialing", "hello", "paired", "backoff" @@ -28,7 +33,7 @@ static char const * const SOURCE_NAMES[] = { }; static char const * const HANDOFF_NAMES[] = { "nothing handed over", "pending", "taken", "declined", "restarted", "cancelled by `failover promote --force`", - "the dialed machine was a standby" + "the dialed machine does not run the identity" }; #define ACTION_NAME_CNT ( sizeof(ACTION_NAMES )/sizeof(ACTION_NAMES [ 0 ]) ) @@ -84,7 +89,7 @@ failover_cmd_args( int * pargc, FD_LOG_ERR(( "could not resolve --address `%s` to an IPv4 address", address )); } if( FD_UNLIKELY( address && !args->failover.addr ) ) { - FD_LOG_ERR(( "--address `%s` is 0.0.0.0, provide the active's own address", address )); + FD_LOG_ERR(( "--address `%s` is 0.0.0.0, provide the address of the machine running the identity", address )); } if( port ) { uint value = 0U; @@ -121,66 +126,118 @@ failover_args( int * pargc, } static char const * -role_name( uchar role ) { - if( FD_LIKELY( role==FD_FAILOVER_ROLE_STANDBY ) ) return "standby"; - if( FD_LIKELY( role==FD_FAILOVER_ROLE_ACTIVE ) ) return "active"; - return "unknown"; +action_name( uchar action ) { + return action`"; case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: return "a transition or key switch is running, check `failover status`"; - case FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS: return "gossip has no address for the active, wait or give it with --address, and if both machines show " - "`role: standby` no machine holds the identity and `failover promote --force` on one of them takes it"; + case FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS: return "gossip has no address for the machine running the identity, wait or give it with --address, and if both machines show " + "`runs target identity: no` no machine runs it and `failover promote --force` on one of them takes it"; case FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY: return "the peer could not finish this request, check `failover status` and the log on the peer"; - case FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED: return "the peer cannot be verified, --force is required, including first use and restart. Vote history is unknown: it has not been checked. Use `failover promote` here to request a handoff from an active failover peer, or `failover promote --force` only after ensuring every other machine with this identity cannot sign"; - case FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE: return "the authenticated peer holds the identity"; - case FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING: return "the peer has not answered our last handoff, if it shows `role: active` run `failover promote` there to " - "finish it, if it shows `role: standby` nothing votes and `failover promote --force` on one machine takes the identity"; + case FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS: return "the authenticated peer runs the identity"; + case FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING: return "the peer has not answered our last handoff, if it shows `runs target identity: yes` run `failover promote` there to " + "finish it, if it shows `runs target identity: no` nothing votes and `failover promote --force` on one machine takes the identity"; case FD_FAILOVER_CONTROL_RESULT_TAKEN: return "the peer took our last handoff and may still be voting"; - case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "gossip showed the staked identity at another host within the last 15 seconds, an active is publishing"; - case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: return "the active has no eligible final vote state to hand over and keeps the identity, check its voting progress and vote-history logs before retrying"; - case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: return "this standby is too far behind the active's vote history and the active keeps the identity, run `failover promote` again once this machine has caught up"; - case FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED: return "provide the active's keypair with `failover promote ` to prepare this client's target identity"; - case FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT: return "the keypair must match the prepared target and must differ from this machine's boot identity"; - case FD_FAILOVER_CONTROL_RESULT_NOT_BOOT_IDENTITY: return "the client must still be using its boot identity when preparing a target"; + case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "gossip showed the identity at another host within the last 15 seconds, it probably runs there"; + case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: return "the machine running the identity has no eligible final vote state to hand over and keeps it, check its voting progress and vote-history logs before retrying"; + case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: return "this machine is too far behind the vote history of the machine running the identity, which keeps it, run `failover promote` again once this machine has caught up"; + case FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT: return "the keypair is this machine's junk key"; case FD_FAILOVER_CONTROL_RESULT_INPUT_LOST: return "failover lost an input update and is disabled until restart, validator processing continues"; - case FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER: return "the target identity is already an authorized voter, remove it from the authorized voters before preparing it"; case FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH: return "the public key in the keypair file does not match its private key"; + case FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER: return "the keypair is an authorized voter on this validator, `remove-all-authorized-voters` first, failover cannot run an identity that also signs votes as an authorized voter"; default: return NULL; } } +/* What this machine knows of another machine running the identity, + from the checks that `failover promote --force` skips. */ +static char const * +other_holder( ulong result ) { + switch( result ) { + case FD_FAILOVER_CONTROL_RESULT_INPUT_LOST: return "unknown after input loss, failover requires a restart"; + case FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER: return "n/a, this machine runs the identity"; + case FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS: return "the authenticated peer says it runs the identity"; + case FD_FAILOVER_CONTROL_RESULT_TAKEN: return "the peer took our last handoff and may be voting"; + case FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING: return "the peer has not answered our last handoff and may run the identity"; + case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "gossip showed the identity at another host within 15 seconds, it is probably voting there, " + "for 15 seconds after a restart this can be an old entry"; + case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: return "unknown while a transition or key switch runs here"; + case FD_ADMINCTL_RESULT_SUCCESS: return "none seen by this machine, which does not prove the other machine is stopped, check it before " + "`failover promote --force`"; + default: return "unknown"; + } +} + +/* The installed identity relative to the target and the junk key. */ +static char const * +installed_name( fd_adminctl_failover_status_resp_t const * resp ) { + if( !memcmp( resp->installed, resp->target, 32UL ) ) return "the target"; + if( !memcmp( resp->installed, resp->junk, 32UL ) ) return "this machine's junk key"; + return "another key"; +} + +/* Prints the identity lines of a status, shared by `failover status` + and the --force prompt. */ +static void +failover_identities_print( fd_adminctl_failover_status_resp_t const * resp ) { + FD_BASE58_ENCODE_32_BYTES( resp->target, target_b58 ); + FD_BASE58_ENCODE_32_BYTES( resp->installed, installed_b58 ); + FD_LOG_STDOUT(( "%-26s %s\n", "target identity:", target_b58 )); + FD_LOG_STDOUT(( "%-26s %s (%s)\n", "installed identity:", installed_b58, installed_name( resp ) )); + FD_LOG_STDOUT(( "%-26s %s\n", "runs target identity:", resp->holds ? "yes" : "no" )); +} + +static void +failover_floor_print( fd_adminctl_failover_status_resp_t const * resp ) { + if( FD_UNLIKELY( resp->promote_floor==FD_FAILOVER_SLOT_NULL ) ) FD_LOG_STDOUT(( "%-26s none known\n", "coverage floor:" )); + else FD_LOG_STDOUT(( "%-26s slot %lu\n", "coverage floor:", resp->promote_floor )); +} + +/* status is the live `failover status`, NULL when it is unavailable. */ static void -failover_confirm( args_t const * args ) { - /* A promote hands off the staked identity to this machine, so the operator - confirms first. */ +failover_confirm( args_t const * args, + fd_adminctl_failover_status_resp_t const * status ) { + /* A promote hands off the identity for this machine, so the operator confirms first. */ if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_PROMOTE ) { - FD_LOG_STDOUT(( "This takes the staked identity on your word. No other machine may hold it or be\n" - "in a promotion, including any validator running this identity without failover.\n" - "Run `failover status` on the other machine: its local role must be standby with\n" - "no promotion in progress, or that machine must be stopped or otherwise unable to\n" - "vote. Never promote both machines at once. Votes the other machine cast that\n" - "this machine is not aware of are not accounted for.\n" - "WARNING: --force skips every check on the other machine. Use it only when that\n" - "machine cannot sign, or both machines may vote with the staked identity.\n" + if( FD_LIKELY( status ) ) { + failover_identities_print( status ); + FD_LOG_STDOUT(( "%-26s %s\n", "other holder:", other_holder( status->promote_result ) )); + failover_floor_print( status ); + FD_LOG_STDOUT(( "force only at most one machine at a time.\n" )); + } + FD_LOG_STDOUT(( "This takes the identity upon your command. No other machine may run it or be in a\n" + "promotion, including any validator running this identity without failover.\n" + "Run `failover status` on the other machine. It must show `runs target identity: no`\n" + "with no promotion in progress, or that machine must be stopped or otherwise\n" + "unable to vote. Never promote both machines at once.\n" + "WARNING: --force skips every check on the other machine. Use it only when the\n" + "other machine is not signing votes, or both machines may double-vote.\n" "Promotion tries eligible saved vote history first, then starts from an EMPTY\n" "history with a bound covering replay and known votes. Missing votes and leader\n" "windows above that bound remain unprotected.\n" )); } if( FD_UNLIKELY( !args->failover.yes && isatty( STDIN_FILENO ) ) ) { if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) { - FD_LOG_STDOUT(( "This asks the active to hand off the staked identity to this standby. The active\n" - "keeps it if this standby is more than 64 slots behind its finality anchor.\n" + FD_LOG_STDOUT(( "This asks the remote validator running this identity to hand it to this machine. It keeps\n" + "the identity if this machine is more than 64 slots behind its finality anchor.\n" "Otherwise, it gives up the identity before this validator adopts its history.\n" "If replay stalls, the handoff can fail with neither machine voting.\n" "Check that this machine is caught up. Type yes to continue: " )); @@ -193,79 +250,49 @@ failover_confirm( args_t const * args ) { } } -/* What this machine knows of another holder of the staked identity, - from the checks that `failover promote --force` skips. */ -static char const * -other_holder( ulong result ) { - switch( result ) { - case FD_FAILOVER_CONTROL_RESULT_INPUT_LOST: return "unknown after input loss, failover requires a restart"; - case FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED: return "no target identity prepared"; - case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: return "n/a, this machine holds the identity"; - case FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE: return "the authenticated peer says it holds the identity"; - case FD_FAILOVER_CONTROL_RESULT_TAKEN: return "the peer took our last handoff and may be voting"; - case FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING: return "the peer has not answered our last handoff and may hold the identity"; - case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "gossip showed the staked identity at another host within 15 seconds, an active is probably voting, " - "for 15 seconds after a restart this can be an old entry"; - case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: return "unknown while a transition or key switch runs here"; - case FD_ADMINCTL_RESULT_SUCCESS: - case FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED: return "none seen by this machine, which does not prove the other machine is stopped, check it before " - "`failover promote --force`"; - default: return "unknown"; - } -} - /* Prints the response to `failover status`. */ static void failover_status_print( fd_adminctl_failover_status_resp_t const * resp ) { if( FD_UNLIKELY( !resp->enabled ) ) { - FD_LOG_STDOUT(( "%-22s %s\n", "failover:", "disabled" )); + FD_LOG_STDOUT(( "%-26s %s\n", "failover:", "disabled" )); return; } - FD_LOG_STDOUT(( "%-22s %s\n", "failover:", resp->enabled==2 ? "client only, listener disabled" : "enabled" )); - FD_LOG_STDOUT(( "%-22s %s\n", "role:", role_name ( resp->role ) )); + FD_LOG_STDOUT(( "%-26s %s\n", "failover:", "enabled" )); + failover_identities_print( resp ); + FD_BASE58_ENCODE_32_BYTES( resp->junk, junk_b58 ); + FD_LOG_STDOUT(( "%-26s %s\n", "junk key:", junk_b58 )); if( FD_UNLIKELY( resp->request_paused ) ) { - FD_LOG_STDOUT(( "%-22s %s, paused at its deadline and not dialing, `failover promote` resumes it\n", "action:", + FD_LOG_STDOUT(( "%-26s %s, paused at its deadline and not dialing, `failover promote` resumes it\n", "action:", action_name( resp->action ) )); } else { - FD_LOG_STDOUT(( "%-22s %s\n", "action:", action_name( resp->action ) )); + FD_LOG_STDOUT(( "%-26s %s\n", "action:", action_name( resp->action ) )); } - FD_LOG_STDOUT(( "%-22s %s\n", "stuck:", !resp->stuck ? "no" : resp->request_paused ? "yes, the handoff request is paused, see the log" : "yes, see the log" )); - FD_LOG_STDOUT(( "%-22s %s\n", "link:", resp->link_statelink_state ] : "unknown" )); - FD_LOG_STDOUT(( "%-22s %s\n", "peer role:", resp->peer_role_valid ? role_name( resp->peer_role ) : "unknown, no authenticated failover session" )); - if( FD_UNLIKELY( !resp->peer_boot_id ) ) FD_LOG_STDOUT(( "%-22s none paired since boot\n", "peer boot id:" )); - else FD_LOG_STDOUT(( "%-22s %016lx\n", "peer boot id:", resp->peer_boot_id )); - if( FD_UNLIKELY( !resp->peer_addr && resp->role==FD_FAILOVER_ROLE_ACTIVE ) ) { - FD_LOG_STDOUT(( "%-22s n/a, this machine is the active\n", "peer address:" )); + FD_LOG_STDOUT(( "%-26s %s\n", "stuck:", !resp->stuck ? "no" : resp->request_paused ? "yes, the handoff request is paused, see the log" : "yes, see the log" )); + FD_LOG_STDOUT(( "%-26s %s\n", "link:", resp->link_statelink_state ] : "unknown" )); + FD_LOG_STDOUT(( "%-26s %s\n", "peer runs target identity:", !resp->peer_holds_valid ? "unknown, no authenticated failover session" : resp->peer_holds ? "yes" : "no" )); + if( FD_UNLIKELY( !resp->peer_boot_id ) ) FD_LOG_STDOUT(( "%-26s none paired since boot\n", "peer boot id:" )); + else FD_LOG_STDOUT(( "%-26s %016lx\n", "peer boot id:", resp->peer_boot_id )); + if( FD_UNLIKELY( !resp->peer_addr && resp->holds ) ) { + FD_LOG_STDOUT(( "%-26s n/a, this machine runs the identity\n", "peer address:" )); } else if( FD_UNLIKELY( !resp->peer_addr ) ) { - FD_LOG_STDOUT(( "%-22s unknown, gossip has no contact info for the staked identity from another host\n", "peer address:" )); + FD_LOG_STDOUT(( "%-26s unknown, gossip has no contact info for the identity from another host\n", "peer address:" )); } else { - FD_LOG_STDOUT(( "%-22s " FD_IP4_ADDR_FMT ":%u, from %s\n", "peer address:", FD_IP4_ADDR_FMT_ARGS( resp->peer_addr ), (uint)resp->peer_port, + FD_LOG_STDOUT(( "%-26s " FD_IP4_ADDR_FMT ":%u, from %s\n", "peer address:", FD_IP4_ADDR_FMT_ARGS( resp->peer_addr ), (uint)resp->peer_port, resp->peer_addr_cmd ? "--address" : "gossip" )); } if( FD_UNLIKELY( !resp->handoff_id ) ) { - FD_LOG_STDOUT(( "%-22s none since boot\n", "last handoff:" )); + FD_LOG_STDOUT(( "%-26s none since boot\n", "last handoff:" )); } else { - FD_LOG_STDOUT(( "%-22s %lu, %s\n", "last handoff:", resp->handoff_id, + FD_LOG_STDOUT(( "%-26s %lu, %s\n", "last handoff:", resp->handoff_id, resp->handoff_resulthandoff_result ] : "unknown" )); } - FD_LOG_STDOUT(( "%-22s %s\n", "other holder:", other_holder( resp->promote_result ) )); - FD_LOG_STDOUT(( "%-22s %s\n", "takeover adopts:", + FD_LOG_STDOUT(( "%-26s %s\n", "other holder:", other_holder( resp->promote_result ) )); + FD_LOG_STDOUT(( "%-26s %s\n", "takeover adopts:", resp->promote_sourcepromote_source ] : "unknown" )); - if( FD_UNLIKELY( resp->promote_floor==FD_FAILOVER_SLOT_NULL ) ) FD_LOG_STDOUT(( "%-22s none known\n", "coverage floor:" )); - else FD_LOG_STDOUT(( "%-22s slot %lu\n", "coverage floor:", resp->promote_floor )); -} - -/* Signal handler that tells the operator a command is still waiting. */ -static void -wait_note( int sig ) { - (void)sig; - static char const note[] = "the validator has not answered for 3 seconds, it may be switching its identity, " - "see `failover: switching the identity` in its log, still waiting\n"; - ssize_t written = write( STDERR_FILENO, note, sizeof(note)-1UL ); - (void)written; + failover_floor_print( resp ); } /* Sends req to the failover tile and waits for the answer, which lands @@ -291,19 +318,13 @@ failover_request( fd_adminctl_t * adminctl, } ulong cmd = req->cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS ? FD_ADMINCTL_CMD_FAILOVER_STATUS : FD_ADMINCTL_CMD_FAILOVER_PROMOTE; fd_adminctl_publish( adminctl, slot_idx, cmd, payload_sz ); - - struct sigaction sa = { .sa_handler = wait_note }; - if( FD_UNLIKELY( sigaction( SIGALRM, &sa, NULL ) ) ) FD_LOG_ERR(( "sigaction(SIGALRM) failed (%i-%s)", errno, fd_io_strerror( errno ) )); - alarm( 3U ); - ulong result = fd_adminctl_wait_response( adminctl, slot_idx, resp, resp_max, resp_sz ); - alarm( 0U ); - return result; + return fd_adminctl_wait_response( adminctl, slot_idx, resp, resp_max, resp_sz ); } static void __attribute__((noreturn)) fail_disabled( int status ) { if( status ) FD_LOG_ERR(( "`failover status` is unavailable because this validator has no failover command bus" )); - FD_LOG_ERR(( "failover is not enabled on the running validator, set [failover.enabled] to true and restart it" )); + FD_LOG_ERR(( "failover is not running on this validator, it needs [paths.vote_account] and [failover.enabled] true, then a restart" )); } static void __attribute__((noreturn)) @@ -311,25 +332,53 @@ fail_incompatible( void ) { FD_LOG_ERR(( "running validator returned an incompatible failover response, use the firedancer binary the validator runs" )); } -/* Before we prompt we check that failover is on and that this machine is - in the role the command needs, so nobody confirms a command that is - then refused. */ -static void -failover_precheck( fd_adminctl_t * adminctl, - args_t const * args ) { - fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_STATUS }; - fd_adminctl_failover_status_resp_t status; - ulong status_sz = 0UL; - ulong result = failover_request( adminctl, &req, NULL, &status, sizeof(status), &status_sz ); +/* Reads the live `failover status` into status before a promote. + Returns 1 when status is valid, 0 when the validator gave no usable + answer. Exits if failover is off. */ +static int +failover_status_fetch( fd_adminctl_t * adminctl, + fd_adminctl_failover_status_resp_t * status ) { + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_STATUS }; + ulong status_sz = 0UL; + ulong result = failover_request( adminctl, &req, NULL, status, sizeof(*status), &status_sz ); if( FD_UNLIKELY( result==FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ) ) fail_disabled( 0 ); - if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS || status_sz!=sizeof(status) || - status.version!=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION ) ) return; - if( FD_UNLIKELY( !status.enabled ) ) fail_disabled( 0 ); - if( FD_UNLIKELY( status.role==FD_FAILOVER_ROLE_ACTIVE && + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS || status_sz!=sizeof(*status) || + status->version!=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION ) ) return 0; + if( FD_UNLIKELY( !status->enabled ) ) fail_disabled( 0 ); + return 1; +} + +/* Before we prompt we check that this machine does not already run + the identity it would take. + A keypair picks a new identity to take. */ +static void FD_FN_SENSITIVE +failover_precheck( fd_adminctl_failover_status_resp_t const * status, + args_t const * args ) { + char const * cmd_name = fd_adminctl_failover_cmd_typed( (ulong)args->failover.cmd ); + if( args->failover.keypair_path[ 0 ] ) { + uchar pubkey[ 32 ]; + fd_sha512_t sha512[ 1 ]; + uchar const * keypair = fd_keyload_load( args->failover.keypair_path, 0 ); + FD_TEST( fd_sha512_join( fd_sha512_new( sha512 ) ) ); + fd_ed25519_public_from_private( pubkey, keypair, sha512 ); + fd_keyload_unload( keypair, 0 ); + int same = !memcmp( pubkey, status->target, 32UL ); + if( FD_UNLIKELY( same && status->holds ) ) { + FD_LOG_ERR(( "`%s` refused, %s", cmd_name, control_result_name( FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER ) )); + } + if( FD_UNLIKELY( !same && status->holds && status->promote_floor!=FD_FAILOVER_SLOT_NULL && !args->failover.force ) ) { + FD_BASE58_ENCODE_32_BYTES( status->target, target_b58 ); + FD_LOG_ERR(( "`%s` refused, %s", cmd_name, holds_target_name( target_b58 ) )); + } + if( FD_UNLIKELY( !same && status->action!=FD_FAILOVER_ACTION_IDLE && !args->failover.force ) ) { + FD_LOG_ERR(( "`%s` refused, a transition is running; wait for `failover status` to show idle, or --force to drop it", cmd_name )); + } + return; + } + if( FD_UNLIKELY( status->holds && !( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF && - status.request_paused && status.action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) ) { - FD_LOG_ERR(( "`%s` refused, %s", fd_adminctl_failover_cmd_typed( (ulong)args->failover.cmd ), - control_result_name( FD_FAILOVER_CONTROL_RESULT_BAD_ROLE ) )); + status->request_paused && status->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) ) { + FD_LOG_ERR(( "`%s` refused, %s", cmd_name, control_result_name( FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER ) )); } } @@ -339,11 +388,14 @@ failover_cmd_fn( args_t * args, config_t * config ) { fd_adminctl_t * adminctl = adminctl_client_attach( config, args->failover.name ); int status = args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_STATUS; - if( !status && !args->failover.yes && isatty( STDIN_FILENO ) ) failover_precheck( adminctl, args ); - if( !status ) failover_confirm( args ); + if( !status ) { + fd_adminctl_failover_status_resp_t live; + int live_ok = failover_status_fetch( adminctl, &live ); + if( live_ok && !args->failover.yes && isatty( STDIN_FILENO ) ) failover_precheck( &live, args ); + failover_confirm( args, live_ok ? &live : NULL ); + } - /* --yes only skips the prompt, the operator still has to give --force - explicitly. */ + /* --yes only skips the prompt, the operator still has to give --force explicitly. */ fd_adminctl_failover_req_t req; fd_memset( &req, 0, sizeof(req) ); req.version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION; @@ -374,13 +426,13 @@ failover_cmd_fn( args_t * args, failover_status_print( &resp.status ); } else { ulong id = resp.control.handoff_id; - FD_LOG_STDOUT(( "%-22s accepted\n", cmd_name )); - FD_LOG_STDOUT(( "%-22s %s\n", "role:", role_name ( resp.control.role ) )); - FD_LOG_STDOUT(( "%-22s %s\n", "action:", action_name( resp.control.action ) )); + FD_LOG_STDOUT(( "%-26s accepted\n", cmd_name )); + FD_LOG_STDOUT(( "%-26s %s\n", "runs target identity:", resp.control.holds ? "yes" : "no" )); + FD_LOG_STDOUT(( "%-26s %s\n", "action:", action_name( resp.control.action ) )); if( args->failover.cmd==(int)FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) { - FD_LOG_STDOUT(( "%-22s %lu, done when `failover status` here shows role: active and last handoff: %lu, taken\n", "handoff:", id, id )); + FD_LOG_STDOUT(( "%-26s %lu, done when `failover status` here shows runs target identity: yes and last handoff: %lu, taken\n", "handoff:", id, id )); } else { - FD_LOG_STDOUT(( "%-22s done when `failover status` here shows role: active and action: idle\n", "promotion:" )); + FD_LOG_STDOUT(( "%-26s done when `failover status` here shows runs target identity: yes and action: idle\n", "promotion:" )); } } break; @@ -391,7 +443,7 @@ failover_cmd_fn( args_t * args, FD_LOG_ERR(( "the validator's failover tile did not answer in time, the outcome is indeterminate, run `failover status`" )); case FD_ADMINCTL_RESULT_UNKNOWN_COMMAND: if( status ) { - FD_LOG_STDOUT(( "%-22s %s\n", "failover:", "disabled" )); + FD_LOG_STDOUT(( "%-26s %s\n", "failover:", "disabled" )); break; } fail_disabled( 0 ); @@ -413,22 +465,17 @@ failover_status_args_help( fd_action_help_t * help ) { static void failover_args_help( fd_action_help_t * help ) { - fd_action_help_arg( help, "", NULL, "Request takeover on this validator, or read its failover state" ); - fd_action_help_arg( help, "[]", NULL, "Path to the target identity keypair, which must match the\n" - "standby's current failover target" ); + fd_action_help_arg( help, "", NULL, "Request takeover on this validator (promote), or read its failover state (status)." ); + fd_action_help_arg( help, "[]", NULL, "Path to the keypair of the identity to take." ); failover_status_args_help( help ); fd_action_help_arg( help, "--yes", NULL, "Skip the confirmation prompt. It does not authorize peer or\n" "vote-history overrides" ); - fd_action_help_arg( help, "--force", NULL, "Take the identity without the active, skip\n" - "every check on the other machine and stop waiting for its answer\n" - "to our handoff. Use it only when the other machine cannot sign.\n" - "Also accept incomplete or empty vote history, earlier lockouts may\n" - "be lost" ); - fd_action_help_arg( help, "--address", "
", "Only without --force. The active to ask, an IPv4\n" - "address or host name, instead of the address gossip shows for it,\n" - "for example when the active listens on a private network" ); - fd_action_help_arg( help, "--port", "", "Only without --force. The active's failover port, 8010 by\n" - "default" ); + fd_action_help_arg( help, "--force", NULL, "Forcibly take the identity without coordinating with the machine running it, skip\n" + "every check on the other machine. Use it only when the other machine cannot sign.\n" ); + fd_action_help_arg( help, "--address", "
", "Only without --force. The machine running the identity, an\n" + "IPv4 address or host name, instead of the address gossip shows\n" + "for it, for example when it listens on a private network" ); + fd_action_help_arg( help, "--port", "", "Only without --force. Its failover port, 8010 by default" ); } static void @@ -447,21 +494,18 @@ action_t fd_action_failover = { .fn = failover_cmd_fn, .require_config = 0, .perm = failover_promote_perm, - .description = "Promote the running standby validator or read its failover state", - .detail = "Hands off the staked identity between two validators. A validator with\n" - "[failover] enabled boots as a standby under a fresh junk identity and\n" - "does not vote until promoted. Both validators must use the same\n" - "[paths.vote_account].\n" + .description = "Take the identity that another validator runs or read the failover status", + .detail = "Hands off an identity between validators. Each boots under its own\n" + "[paths.identity_key], and all validators must use the same [paths.vote_account].\n" + "On the machine that should take over, pass the keypair of the target identity.\n" "\n" - "Asks the active to hand over its identity. With --force, takes the\n" - "identity without the active, for when it cannot sign. Both forms\n" - "ask for confirmation in a terminal unless --yes is given.\n" - "Outside a terminal they proceed without a prompt. Use `failover status`\n" - "to check progress.\n" + "Asks the machine running the identity to hand it over. With --force,\n" + "takes the identity without handover.\n" + "This command asks for confirmation unless --yes is given.\n" + "Use `failover status` to check handover progress.\n" "\n" "This command does not start a validator. It attaches to one that is already\n" - "running. With no arguments it discovers the running validator automatically.\n" - "If multiple validators are running, pass --name to select one.\n", + "running. If multiple validators are running, pass --name to select one.\n", .usage = "failover promote [] [--name ] [--yes] [--force] [--address
] [--port ]\n" " firedancer failover status [--name ]", .args_help = failover_args_help, diff --git a/src/app/firedancer/commands/set_identity.c b/src/app/firedancer/commands/set_identity.c index 358c5681e22..4ec132aea9b 100644 --- a/src/app/firedancer/commands/set_identity.c +++ b/src/app/firedancer/commands/set_identity.c @@ -132,13 +132,13 @@ set_identity( args_t * args, case FD_SET_IDENTITY_RESULT_INVALID_VOTE_HISTORY: FD_LOG_ERR(( "Failed to set identity: the vote history file is invalid or does not belong to the new identity. " "If you believe the vote history file is valid, please contact the Firedancer team for additional assistance." )); - case FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER: - FD_LOG_ERR(( "Failed to set identity: the new failover identity is already an authorized voter. " - "Remove it from the authorized voters before selecting it as the identity." )); + case FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH: + FD_LOG_ERR(( "Failed to set identity: the public key in the keypair does not match the public key derived " + "from its private key." )); case FD_ADMINCTL_RESULT_UNKNOWN_COMMAND: + case FD_ADMINCTL_RESULT_UNSUPPORTED: case FD_ADMINCTL_RESULT_ABI_VERSION_MISMATCH: case FD_ADMINCTL_RESULT_ABI_SIZE_MISMATCH: - case FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH: FD_LOG_ERR(( "Failed to set identity: the command was not able to successfully communicate " "with the running Firedancer process. It is possible that you are running the " "command from an older or newer version of Firedancer that is no longer compatible." )); diff --git a/src/app/firedancer/config/default.toml b/src/app/firedancer/config/default.toml index 653e5a5f099..532298c2ba8 100644 --- a/src/app/firedancer/config/default.toml +++ b/src/app/firedancer/config/default.toml @@ -103,6 +103,11 @@ telemetry = true # if one is not provided (or has not already been generated) and # place it in this path. If no path is provided, it is defaulted # to inside the base directory, under path `identity.json`. + # + # The validator boots under this identity. For failover, a standby + # machine that waits to take over another validator's identity + # should boot under an unstaked randomized key, created using + # `firedancer keys new`. identity_key = "" # Absolute path to a `keypair.json` containing the voting account @@ -1991,16 +1996,17 @@ telemetry = true # history files are only for handing off an identity from a validator that # does not run failover, such as Agave. [failover] - # Whether to enable failover. When true, the validator boots as - # standby under a fresh junk identity generated in memory. Its - # private key is not written to a file. When false, it boots - # normally under [paths.identity_key]. - enabled = false + # Whether to enable failover. It only takes effect with + # [paths.vote_account] set. The validator boots under + # [paths.identity_key], and when `failover promote ` is + # run on a standby host, it takes over the identity in that keypair + # from the machine that is actively running it. + enabled = true # The address to listen on. If enabled, by default the listener is # exposed to the network. This is safe as possession of the private - # key is required to demote the node, but could be a vector for - # a targeted denial of service. For added reliability, restrict + # key is required to take over the identity, but could be a vector + # for a targeted denial of service. For added reliability, restrict # the listener to a private network. listen_address = "0.0.0.0" diff --git a/src/app/firedancer/topology.c b/src/app/firedancer/topology.c index 976e7add201..7ed69ecad3c 100644 --- a/src/app/firedancer/topology.c +++ b/src/app/firedancer/topology.c @@ -1,5 +1,4 @@ #include "topology.h" -#include "../../disco/keyguard/fd_keyload.h" #include "../shared/fd_config_json.h" #include "../../discof/poh/fd_poh.h" @@ -286,7 +285,7 @@ fd_topo_initialize( config_t * config ) { int leader_enabled = !!config->firedancer.layout.enable_block_production; int rserve_enabled = config->tiles.rserve.enabled; int efficient_mode = !strcmp( config->firedancer.layout.mode, "efficient" ); - int failover_enabled = config->firedancer.failover.enabled || config->paths.vote_account[ 0 ]; + int failover_enabled = config->firedancer.failover.enabled && config->paths.vote_account[ 0 ]; if( FD_UNLIKELY( snapmk_enabled ) ) { FD_CHECK_ERR( config->firedancer.snapshots.max_full_snapshots_to_keep, @@ -321,7 +320,7 @@ fd_topo_initialize( config_t * config ) { fd_topob_wksp( topo, "admin" )->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_NEVER; /* The failov tile keeps TLS session keys in its scratch, so its workspace stays out of core dumps like admin and sign. */ - if( FD_UNLIKELY( failover_enabled ) ) fd_topob_wksp( topo, "failov" )->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_NEVER; + if( FD_LIKELY( failover_enabled ) ) fd_topob_wksp( topo, "failov" )->core_dump_level = FD_TOPO_CORE_DUMP_LEVEL_NEVER; if( leader_enabled ) { fd_topob_wksp( topo, "quic" ); @@ -409,14 +408,14 @@ fd_topo_initialize( config_t * config ) { fd_topob_wksp( topo, "votor_sign" ); fd_topob_wksp( topo, "sign_votor" ); - if( FD_UNLIKELY( failover_enabled ) ) { + if( FD_LIKELY( failover_enabled ) ) { fd_topob_wksp( topo, "failov_sign" ); fd_topob_wksp( topo, "sign_failov" ); } fd_topob_wksp( topo, "execrp_replay" ); fd_topob_wksp( topo, "admin_replay" ); - if( FD_UNLIKELY( failover_enabled ) ) { + if( FD_LIKELY( failover_enabled ) ) { fd_topob_wksp( topo, "admin_failov" ); fd_topob_wksp( topo, "failov_admin" ); fd_topob_wksp( topo, "failov_votor" ); @@ -525,10 +524,10 @@ fd_topo_initialize( config_t * config ) { FOR(execrp_tile_cnt) fd_topob_link( topo, "replay_execrp", "replay_execrp", 256UL, sizeof(fd_execrp_task_msg_t), 1UL ); /**/ fd_topob_link( topo, "admin_replay", "admin_replay", 32UL, 0UL, 1UL ); /**/ fd_topob_link( topo, "replay_admin", "admin_replay", 32UL, 0UL, 1UL ); - if( FD_UNLIKELY( failover_enabled ) ) { + if( FD_LIKELY( failover_enabled ) ) { /**/ fd_topob_link( topo, "admin_failov", "admin_failov", 32UL, FD_FAILOVER_BUS_MTU, 1UL ); /**/ fd_topob_link( topo, "failov_admin", "failov_admin", 32UL, FD_FAILOVER_BUS_MTU, 1UL ); - /**/ fd_topob_link( topo, "failov_votor", "failov_votor", 32UL, FD_FAILOVER_STATE_MAX, 1UL ); + /**/ fd_topob_link( topo, "failov_votor", "failov_votor", 32UL, FD_VOTOR_ADOPT_REQ_HDR_SZ+FD_FAILOVER_STATE_MAX, 1UL ); /**/ fd_topob_link( topo, "votor_failov", "failov_votor", 32UL, sizeof(fd_votor_adopt_result_t), 1UL ); /**/ fd_topob_link( topo, "votor_hist", "votor_hist", 1024UL, sizeof(fd_votor_hist_msg_t), 1UL ); if( !config->tiles.gui.enabled ) { @@ -559,7 +558,7 @@ fd_topo_initialize( config_t * config ) { FOR(sign_tile_cnt-1) fd_topob_link( topo, "repair_sign", "repair_sign", 256UL, FD_REPAIR_SIG_SER_MAX, 1UL ); /* See repair_tile.c for explanation */ FOR(sign_tile_cnt-1) fd_topob_link( topo, "sign_repair", "sign_repair", 256UL, sizeof(fd_ed25519_sig_t), 1UL ); - if( FD_UNLIKELY( failover_enabled ) ) { + if( FD_LIKELY( failover_enabled ) ) { /**/ fd_topob_link( topo, "failov_sign", "failov_sign", 128UL, 130UL, 1UL ); /**/ fd_topob_link( topo, "sign_failov", "sign_failov", 128UL, sizeof(fd_ed25519_sig_t), 1UL ); } @@ -778,7 +777,7 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_tile_out( topo, "replay", 0UL, "replay_admin", 0UL ); /**/ fd_topob_tile_out( topo, "admin", 0UL, "admin_replay", 0UL ); /**/ fd_topob_tile_in ( topo, "admin", 0UL, "metric_in", "replay_admin", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); - if( FD_UNLIKELY( failover_enabled ) ) { + if( FD_LIKELY( failover_enabled ) ) { if( !config->tiles.gui.enabled ) { fd_topob_tile_out( topo, "gossip", 0UL, "gossip_ciseen", 0UL ); } @@ -933,7 +932,7 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_rserve", 0UL ); } - if( FD_UNLIKELY( failover_enabled ) ) { + if( FD_LIKELY( failover_enabled ) ) { /**/ fd_topob_tile_in ( topo, "sign", 0UL, "metric_in", "failov_sign", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "sign", 0UL, "sign_failov", 0UL ); } @@ -1089,31 +1088,29 @@ fd_topo_initialize( config_t * config ) { } } - if( FD_UNLIKELY( failover_enabled ) ) { - /**/ fd_topob_tile( topo, "failov", "failov", "metric_in", ULONG_MAX, 0, 0, 0 ); + if( FD_LIKELY( failover_enabled ) ) { + /**/ fd_topob_tile( topo, "failov", "failov", "metric_in", tile_to_cpu[ topo->tile_cnt ], 0, 0, 0 ); - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "gossip_ciseen", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "gossip_ciseen", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "admin_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_admin", 0UL ); - /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_hist", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); + /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_hist", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_votor", 0UL ); /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "votor_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "failov", 0UL, "failov_sign", 0UL ); /**/ fd_topob_tile_in ( topo, "failov", 0UL, "metric_in", "sign_failov", 0UL, FD_TOPOB_UNRELIABLE, FD_TOPOB_UNPOLLED ); } - ulong pinned_tile_cnt = topo->tile_cnt - (ulong)!!failover_enabled; - if( FD_LIKELY( !is_auto_affinity ) ) { - if( FD_UNLIKELY( affinity_tile_cnttile_cnt ) ) FD_LOG_ERR(( "The topology you are using has %lu tiles, but the CPU affinity specified in the config tile as [layout.affinity] only provides for %lu cores. " "You should either increase the number of cores dedicated to Firedancer in the affinity string, or decrease the number of cores needed by reducing " "the total tile count. You can reduce the tile count by decreasing individual tile counts in the [layout] section of the configuration file.", - pinned_tile_cnt, affinity_tile_cnt )); - if( FD_UNLIKELY( affinity_tile_cnt>pinned_tile_cnt ) ) + topo->tile_cnt, affinity_tile_cnt )); + if( FD_UNLIKELY( affinity_tile_cnt>topo->tile_cnt ) ) FD_LOG_WARNING(( "The topology you are using has %lu tiles, but the CPU affinity specified in the config tile as [layout.affinity] provides for %lu cores. " "Not all cores in the affinity will be used by Firedancer. You may wish to increase the number of tiles in the system by increasing " "individual tile counts in the [layout] section of the configuration file.", - pinned_tile_cnt, affinity_tile_cnt )); + topo->tile_cnt, affinity_tile_cnt )); } else { ushort blocklist_cores[ FD_TILE_MAX ]; topo->blocklist_cores_cnt = fd_topob_parse_affinity_cstr( config->layout.blocklist_cores, blocklist_cores, 0, 0 ); @@ -1162,7 +1159,7 @@ fd_topo_initialize( config_t * config ) { /**/ fd_topob_tile_in ( topo, "rpc", 0UL, "metric_in", "votor_out", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); } - if( FD_UNLIKELY( failover_enabled ) ) { + if( FD_LIKELY( failover_enabled ) ) { /**/ fd_topob_tile_in ( topo, "votor", 0UL, "metric_in", "failov_votor", 0UL, FD_TOPOB_RELIABLE, FD_TOPOB_POLLED ); /**/ fd_topob_tile_out( topo, "votor", 0UL, "votor_hist", 0UL ); /**/ fd_topob_tile_out( topo, "votor", 0UL, "votor_failov", 0UL ); @@ -1433,10 +1430,6 @@ wfs_decode_bank_hash( fd_config_t const * config, void fd_topo_configure_tile( fd_topo_tile_t * tile, fd_config_t * config ) { - char const * identity_key_path = config->firedancer.failover.enabled && fd_topo_find_tile( &config->topo, "failov", 0UL )!=ULONG_MAX - ? FD_KEYLOAD_FAILOVER_JUNK - : config->paths.identity_key; - if( FD_UNLIKELY( !strcmp( tile->name, "metric" ) ) ) { if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->tiles.metric.prometheus_listen_address, &tile->metric.prometheus_listen_addr ) ) ) @@ -1445,7 +1438,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "event" ) ) ) { - fd_cstr_ncpy( tile->event.identity_key_path, identity_key_path, sizeof(tile->event.identity_key_path) ); + fd_cstr_ncpy( tile->event.identity_key_path, config->paths.identity_key, sizeof(tile->event.identity_key_path) ); fd_cstr_ncpy( tile->event.url, config->tiles.event.url, sizeof(tile->event.url) ); fd_cstr_ncpy( tile->event.action, config->action, sizeof(tile->event.action) ); FD_TEST( fd_cstr_printf_check( tile->event.accounts_path, sizeof(tile->event.accounts_path), NULL, "%s", config->paths.accounts ) ); @@ -1508,14 +1501,13 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "admin" ) ) ) { - fd_cstr_ncpy( tile->admin.identity_key_path, identity_key_path, sizeof(tile->admin.identity_key_path) ); + fd_cstr_ncpy( tile->admin.identity_key_path, config->paths.identity_key, sizeof(tile->admin.identity_key_path) ); } else if( FD_UNLIKELY( !strcmp( tile->name, "failov" ) ) ) { - fd_cstr_ncpy( tile->failov.identity_key_path, identity_key_path, sizeof(tile->failov.identity_key_path) ); - fd_cstr_ncpy( tile->failov.staked_key_path, config->firedancer.failover.enabled ? config->paths.identity_key : "", sizeof(tile->failov.staked_key_path) ); - fd_cstr_ncpy( tile->failov.vote_account_path, config->paths.vote_account, sizeof(tile->failov.vote_account_path) ); - if( FD_UNLIKELY( config->firedancer.failover.enabled && !fd_cstr_to_ip4_addr( config->firedancer.failover.listen_address, &tile->failov.listen_addr ) ) ) + fd_cstr_ncpy( tile->failov.identity_key_path, config->paths.identity_key, sizeof(tile->failov.identity_key_path) ); + fd_cstr_ncpy( tile->failov.vote_account_path, config->paths.vote_account, sizeof(tile->failov.vote_account_path) ); + if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( config->firedancer.failover.listen_address, &tile->failov.listen_addr ) ) ) FD_LOG_ERR(( "[failover.listen_address] `%s` must be an IPv4 address, host names are not accepted", config->firedancer.failover.listen_address )); tile->failov.port = config->firedancer.failover.listen_port; fd_cstr_ncpy( tile->failov.gossip_host, config->firedancer.gossip.host, sizeof(tile->failov.gossip_host) ); @@ -1524,7 +1516,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "gossvf") ) ) { - fd_cstr_ncpy( tile->gossvf.identity_key_path, identity_key_path, sizeof(tile->gossvf.identity_key_path) ); + fd_cstr_ncpy( tile->gossvf.identity_key_path, config->paths.identity_key, sizeof(tile->gossvf.identity_key_path) ); tile->gossvf.tcache_depth = 1<<22UL; /* TODO: user defined option */ tile->gossvf.shred_version = 0U; tile->gossvf.allow_private_address = config->development.gossip.allow_private_address; @@ -1548,7 +1540,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->gossip.net_ip_addr = config->net.ip_addr; tile->gossip.ip_addr = config->net.ip_addr; tile->gossip.bind_ip_addr = config->net.ip_addr; - fd_cstr_ncpy( tile->gossip.identity_key_path, identity_key_path, sizeof(tile->gossip.identity_key_path) ); + fd_cstr_ncpy( tile->gossip.identity_key_path, config->paths.identity_key, sizeof(tile->gossip.identity_key_path) ); tile->gossip.expected_shred_version = config->consensus.expected_shred_version; tile->gossip.max_entries = config->tiles.gossip.max_entries; tile->gossip.boot_timestamp_nanos = config->boot_timestamp_nanos; @@ -1643,13 +1635,13 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } } tile->rotor.repair_sign_cnt = config->firedancer.layout.sign_tile_count - 1; /* -1 because this excludes the keyguard client */ - fd_cstr_ncpy( tile->rotor.identity_key_path, identity_key_path, sizeof(tile->rotor.identity_key_path) ); + fd_cstr_ncpy( tile->rotor.identity_key_path, config->paths.identity_key, sizeof(tile->rotor.identity_key_path) ); } else if( FD_UNLIKELY( !strcmp( tile->name, "rserve" ) ) ) { tile->rserve.repair_serve_listen_port = config->tiles.rserve.repair_serve_listen_port; tile->rserve.max_shreds_per_block = config->limits.max_shreds_per_block; tile->rserve.ping_cache_entries = 1UL<<16; /* TODO: Configure this from some global metric? */ tile->rserve.blockdb_max = 1UL<<15; /* ~20.6 KiB each. TODO: make configurable */ - fd_cstr_ncpy( tile->rserve.identity_key_path, identity_key_path, sizeof(tile->rserve.identity_key_path) ); + fd_cstr_ncpy( tile->rserve.identity_key_path, config->paths.identity_key, sizeof(tile->rserve.identity_key_path) ); } else if( FD_UNLIKELY( !strcmp( tile->name, "replay" ) )) { @@ -1665,7 +1657,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->replay.txncache_obj_id = fd_pod_query_ulong( config->topo.props, "txncache", ULONG_MAX ); FD_TEST( tile->replay.txncache_obj_id !=ULONG_MAX ); - fd_cstr_ncpy( tile->replay.identity_key_path, identity_key_path, sizeof(tile->replay.identity_key_path) ); + fd_cstr_ncpy( tile->replay.identity_key_path, config->paths.identity_key, sizeof(tile->replay.identity_key_path) ); tile->replay.ip_addr = config->net.ip_addr; fd_cstr_ncpy( tile->replay.vote_account_path, config->paths.vote_account, sizeof(tile->replay.vote_account_path) ); @@ -1730,8 +1722,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->votor.quic_server_listen_port = config->firedancer.development.votor.quic_server_listen_port; tile->votor.ip_addr = config->net.ip_addr; tile->votor.max_live_slots = config->firedancer.runtime.max_live_slots; - fd_cstr_ncpy( tile->votor.identity_key_path, identity_key_path, sizeof(tile->votor.identity_key_path) ); - tile->votor.failover_standby = config->firedancer.failover.enabled; + fd_cstr_ncpy( tile->votor.identity_key_path, config->paths.identity_key, sizeof(tile->votor.identity_key_path) ); fd_cstr_ncpy( tile->votor.vote_history_path, config->paths.vote_history, sizeof(tile->votor.vote_history_path) ); tile->votor.target_gid = config->gid; tile->votor.target_uid = config->uid; @@ -1816,7 +1807,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, PARSE_BUNDLE_PUBKEY( pack, tip_payment_program_addr ); PARSE_BUNDLE_PUBKEY( pack, tip_distribution_authority ); tile->pack.bundle.commission_bps = config->tiles.bundle.commission_bps; - fd_cstr_ncpy( tile->pack.bundle.identity_key_path, identity_key_path, sizeof(tile->pack.bundle.identity_key_path) ); + fd_cstr_ncpy( tile->pack.bundle.identity_key_path, config->paths.identity_key, sizeof(tile->pack.bundle.identity_key_path) ); fd_cstr_ncpy( tile->pack.bundle.vote_account_path, config->paths.vote_account, sizeof(tile->pack.bundle.vote_account_path) ); } else { fd_memset( &tile->pack.bundle, '\0', sizeof(tile->pack.bundle) ); @@ -1831,14 +1822,14 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->execle.report_runtime_diffs = config->development.event.report_runtime_diffs; } else if( FD_UNLIKELY( !strcmp( tile->name, "motor" ) ) ) { - fd_cstr_ncpy( tile->motor.identity_key_path, identity_key_path, sizeof(tile->motor.identity_key_path) ); + fd_cstr_ncpy( tile->motor.identity_key_path, config->paths.identity_key, sizeof(tile->motor.identity_key_path) ); tile->motor.execle_cnt = config->firedancer.layout.execle_tile_count; tile->motor.max_txn_per_slot = config->limits.max_txn_per_slot; } else if( FD_UNLIKELY( !strcmp( tile->name, "shred" ) ) ) { - fd_cstr_ncpy( tile->shred.identity_key_path, identity_key_path, sizeof(tile->shred.identity_key_path) ); + fd_cstr_ncpy( tile->shred.identity_key_path, config->paths.identity_key, sizeof(tile->shred.identity_key_path) ); tile->shred.fec_exposure = FD_SHRED_FIREDANCER_FEC_EXPOSURE; tile->shred.fec_resolver_depth = config->tiles.shred.max_pending_shred_sets; @@ -1856,10 +1847,9 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, } else if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) ) ) { - fd_cstr_ncpy( tile->sign.identity_key_path, identity_key_path, sizeof(tile->sign.identity_key_path) ); - tile->sign.identity_key_fd = -1; + fd_cstr_ncpy( tile->sign.identity_key_path, config->paths.identity_key, sizeof(tile->sign.identity_key_path) ); + tile->sign.failover_junk_fd = -1; tile->sign.failover_enabled = fd_topo_find_tile( &config->topo, "failov", 0UL )!=ULONG_MAX; - fd_cstr_ncpy( tile->sign.failover_staked_key_path, config->firedancer.failover.enabled ? config->paths.identity_key : "", sizeof(tile->sign.failover_staked_key_path) ); tile->sign.authorized_voter_paths_cnt = config->firedancer.paths.authorized_voter_paths_cnt; for( ulong i=0UL; isign.authorized_voter_paths_cnt; i++ ) { @@ -1896,7 +1886,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->gui.max_txn_per_slot = config->limits.max_txn_per_slot; tile->gui.is_voting = strcmp( config->paths.vote_account, "" ); fd_cstr_ncpy( tile->gui.cluster, config->cluster, sizeof(tile->gui.cluster) ); - fd_cstr_ncpy( tile->gui.identity_key_path, identity_key_path, sizeof(tile->gui.identity_key_path) ); + fd_cstr_ncpy( tile->gui.identity_key_path, config->paths.identity_key, sizeof(tile->gui.identity_key_path) ); fd_cstr_ncpy( tile->gui.vote_key_path, config->paths.vote_account, sizeof(tile->gui.vote_key_path) ); fd_cstr_ncpy( tile->gui.accounts_database_path, config->paths.accounts, sizeof(tile->gui.accounts_database_path) ); fd_cstr_ncpy( tile->gui.gui_database_path, config->paths.guidb, sizeof(tile->gui.gui_database_path) ); @@ -1931,7 +1921,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->rpc.accdb_epoch_fseq_obj_id = fd_pod_query_ulong( config->topo.props, "accdb_epoch.rpc", ULONG_MAX ); FD_TEST( tile->rpc.accdb_epoch_fseq_obj_id!=ULONG_MAX ); - fd_cstr_ncpy( tile->rpc.identity_key_path, identity_key_path, sizeof(tile->rpc.identity_key_path) ); + fd_cstr_ncpy( tile->rpc.identity_key_path, config->paths.identity_key, sizeof(tile->rpc.identity_key_path) ); tile->rpc.snapshot_server_enabled = fd_topo_find_tile( &config->topo, "snapsv", 0UL )!=ULONG_MAX; if( FD_LIKELY( tile->rpc.snapshot_server_enabled ) ) { @@ -1957,7 +1947,7 @@ fd_topo_configure_tile( fd_topo_tile_t * tile, tile->bundle.url_len = strnlen( tile->bundle.url, sizeof(tile->bundle.url)-1UL ); fd_cstr_ncpy( tile->bundle.sni, config->tiles.bundle.tls_domain_name, sizeof(tile->bundle.sni) ); tile->bundle.sni_len = strnlen( tile->bundle.sni, sizeof(tile->bundle.sni)-1UL ); - fd_cstr_ncpy( tile->bundle.identity_key_path, identity_key_path, sizeof(tile->bundle.identity_key_path) ); + fd_cstr_ncpy( tile->bundle.identity_key_path, config->paths.identity_key, sizeof(tile->bundle.identity_key_path) ); fd_cstr_ncpy( tile->bundle.key_log_path, config->development.bundle.ssl_key_log_file, sizeof(tile->bundle.key_log_path) ); tile->bundle.buf_sz = config->development.bundle.buffer_size_kib<<10; tile->bundle.out_depth = config->tiles.verify.receive_buffer_size; diff --git a/src/app/shared/commands/run/run.c b/src/app/shared/commands/run/run.c index d73910dcf4c..c42559a13e9 100644 --- a/src/app/shared/commands/run/run.c +++ b/src/app/shared/commands/run/run.c @@ -81,7 +81,7 @@ run_cmd_perm( args_t * args, fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for serving metrics" ); if( FD_UNLIKELY( config->tiles.gui.gui_listen_port<1024 ) ) fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for serving the GUI" ); - if( FD_UNLIKELY( config->firedancer.failover.enabled && config->firedancer.failover.listen_port<1024 ) ) + if( FD_UNLIKELY( config->firedancer.failover.enabled && config->paths.vote_account[ 0 ] && config->firedancer.failover.listen_port<1024 ) ) fd_cap_chk_cap( chk, NAME, CAP_NET_BIND_SERVICE, "call `bind(2)` to bind to a privileged port for the failover listener" ); } @@ -278,9 +278,9 @@ execve_tile( char const * name, pid_t child = fork(); if( FD_UNLIKELY( -1==child ) ) FD_LOG_ERR(( "fork() failed (%i-%s)", errno, fd_io_strerror( errno ) )); if( FD_LIKELY( !child ) ) { - if( !strcmp( tile->name, "sign" ) && !strcmp( tile->sign.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) { - if( FD_UNLIKELY( -1==fcntl( tile->sign.identity_key_fd, F_SETFD, 0 ) ) ) - FD_LOG_ERR(( "passing the boot identity pipe failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( !strcmp( tile->name, "sign" ) && tile->sign.failover_junk_fd!=-1 ) { + if( FD_UNLIKELY( -1==fcntl( tile->sign.failover_junk_fd, F_SETFD, 0 ) ) ) + FD_LOG_ERR(( "passing the failover junk key pipe failed (%i-%s)", errno, fd_io_strerror( errno ) )); } char _current_executable_path[ PATH_MAX ]; @@ -547,8 +547,8 @@ main_pid_namespace( void * _args ) { for( ulong i=0UL; itopo.tile_cnt; i++ ) { fd_topo_tile_t const * tile = &config->topo.tiles[ i ]; - if( strcmp( tile->name, "sign" ) || strcmp( tile->sign.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) continue; - if( FD_UNLIKELY( close( tile->sign.identity_key_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( strcmp( tile->name, "sign" ) || tile->sign.failover_junk_fd==-1 ) continue; + if( FD_UNLIKELY( close( tile->sign.failover_junk_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); } if( FD_UNLIKELY( close( config_memfd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); @@ -732,8 +732,8 @@ clone_firedancer( config_t const * config, if( FD_UNLIKELY( pid_namespace<0 ) ) FD_LOG_ERR(( "clone() failed (%i-%s)", errno, fd_io_strerror( errno ) )); for( ulong i=0UL; itopo.tile_cnt; i++ ) { fd_topo_tile_t const * tile = &config->topo.tiles[ i ]; - if( strcmp( tile->name, "sign" ) || strcmp( tile->sign.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) continue; - if( FD_UNLIKELY( close( tile->sign.identity_key_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( strcmp( tile->name, "sign" ) || tile->sign.failover_junk_fd==-1 ) continue; + if( FD_UNLIKELY( close( tile->sign.failover_junk_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); } if( FD_UNLIKELY( close( pipefd[ 1 ] ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); @@ -1091,21 +1091,21 @@ static void FD_FN_SENSITIVE failover_junk_prepare( config_t * config ) { if( fd_topo_find_tile( &config->topo, "failov", 0UL )==ULONG_MAX ) return; uchar const * key = fd_keyload_generate(); - fd_memcpy( config->topo.boot_identity, key+32UL, 32UL ); + fd_memcpy( config->topo.failover_junk, key+32UL, 32UL ); for( ulong i=0UL; itopo.tile_cnt; i++ ) { fd_topo_tile_t * tile = &config->topo.tiles[ i ]; if( strcmp( tile->name, "sign" ) ) continue; - FD_TEST( !strcmp( tile->sign.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ); + FD_TEST( tile->sign.failover_enabled && tile->sign.failover_junk_fd==-1 ); int pipefd[ 2 ]; if( FD_UNLIKELY( pipe2( pipefd, O_CLOEXEC ) ) ) FD_LOG_ERR(( "pipe2() failed (%i-%s)", errno, fd_io_strerror( errno ) )); for( ulong off=0UL; off<64UL; ) { long n = write( pipefd[ 1 ], key+off, 64UL-off ); if( FD_UNLIKELY( n==-1L && errno==EINTR ) ) continue; - if( FD_UNLIKELY( n<=0L ) ) FD_LOG_ERR(( "could not deliver the generated boot identity" )); + if( FD_UNLIKELY( n<=0L ) ) FD_LOG_ERR(( "could not deliver the failover junk key" )); off += (ulong)n; } if( FD_UNLIKELY( close( pipefd[ 1 ] ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); - tile->sign.identity_key_fd = pipefd[ 0 ]; + tile->sign.failover_junk_fd = pipefd[ 0 ]; } fd_keyload_unload( key, 0 ); } @@ -1116,11 +1116,12 @@ run_firedancer_init( config_t * config, int check_configure ) { struct stat st; int err = stat( config->paths.identity_key, &st ); - if( FD_UNLIKELY( -1==err && errno==ENOENT ) ) FD_LOG_ERR(( "[consensus.identity_path] key does not exist `%s`. You can generate an identity key at this path by running `%s keys new %s --config `", config->paths.identity_key, FD_BINARY_NAME, config->paths.identity_key )); - else if( FD_UNLIKELY( -1==err ) ) FD_LOG_ERR(( "could not stat [consensus.identity_path] `%s` (%i-%s)", config->paths.identity_key, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( -1==err && errno==ENOENT ) ) FD_LOG_ERR(( "[paths.identity_key] key does not exist `%s`. You can generate an identity key at this path by running `%s keys new %s --config `", config->paths.identity_key, FD_BINARY_NAME, config->paths.identity_key )); + else if( FD_UNLIKELY( -1==err ) ) FD_LOG_ERR(( "could not stat [paths.identity_key] `%s` (%i-%s)", config->paths.identity_key, errno, fd_io_strerror( errno ) )); - /* Failover boots under a fresh identity kept only in memory. */ - if( FD_UNLIKELY( config->firedancer.failover.enabled ) ) failover_junk_prepare( config ); + /* Failover hands the identity off to a fresh junk key kept only in + memory. */ + failover_junk_prepare( config ); /* FIXME: fdctl_check_configure unconditionally checks for network stack prerequisites even if the command being run does not diff --git a/src/app/shared/commands/run/run1.c b/src/app/shared/commands/run/run1.c index ad0a52ccc6b..e43c7afa87f 100644 --- a/src/app/shared/commands/run/run1.c +++ b/src/app/shared/commands/run/run1.c @@ -2,7 +2,6 @@ #include "run.h" #include "../../../../util/tile/fd_tile_private.h" -#include "../../../../disco/keyguard/fd_keyload.h" #include #include /* strtoul */ @@ -94,8 +93,8 @@ run1_cmd_fn( args_t * args, int flags = config->development.sandbox ? CLONE_NEWPID : 0; pid_t clone_pid = clone( tile_main, (uchar *)stack + FD_TILE_PRIVATE_STACK_SZ, flags, &clone_args ); if( FD_UNLIKELY( clone_pid<0 ) ) FD_LOG_ERR(( "clone() failed (%i-%s)", errno, fd_io_strerror( errno ) )); - if( !strcmp( tile->name, "sign" ) && !strcmp( tile->sign.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) { - if( FD_UNLIKELY( close( tile->sign.identity_key_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( !strcmp( tile->name, "sign" ) && tile->sign.failover_junk_fd!=-1 ) { + if( FD_UNLIKELY( close( tile->sign.failover_junk_fd ) ) ) FD_LOG_ERR(( "close() failed (%i-%s)", errno, fd_io_strerror( errno ) )); } if( FD_LIKELY( args->run1.pipe_fd!=-1 ) ) { diff --git a/src/app/shared/fd_config.c b/src/app/shared/fd_config.c index 01125878096..1c75c1315c5 100644 --- a/src/app/shared/fd_config.c +++ b/src/app/shared/fd_config.c @@ -343,12 +343,6 @@ fd_config_fill( fd_config_t * config, replace( config->paths.base, "{user}", config->user ); replace( config->paths.base, "{name}", config->name ); - /* With failover, [paths.identity_key] is the staked key both machines - share. */ - if( FD_UNLIKELY( config->firedancer.failover.enabled && !strcmp( config->paths.identity_key, "" ) ) ) { - FD_LOG_ERR(( "[failover.enabled] is true, so [paths.identity_key] must specify the staked identity keypair both machines share" )); - } - if( FD_UNLIKELY( !strcmp( config->paths.identity_key, "" ) ) ) { /* Development binaries generate an identity key on boot. */ if( FD_UNLIKELY( config->is_live_cluster && !dev ) ) FD_LOG_ERR(( "configuration file must specify [paths.identity_key] when joining a live cluster" )); @@ -537,9 +531,6 @@ fd_config_validate_failover( fd_config_t const * config ) { if( FD_UNLIKELY( !fd_cstr_to_ip4_addr( f->failover.listen_address, &listen_addr ) ) ) { FD_LOG_ERR(( "[failover.listen_address] `%s` must be an IPv4 address, host names are not accepted", f->failover.listen_address )); } - if( FD_UNLIKELY( !strcmp( config->paths.vote_account, "" ) ) ) { - FD_LOG_ERR(( "[failover.enabled] is true, so [paths.vote_account] must be set to the vote account both machines vote for" )); - } } void diff --git a/src/app/shared/fd_config_json.c b/src/app/shared/fd_config_json.c index b8ecc42384f..e58dba64c0f 100644 --- a/src/app/shared/fd_config_json.c +++ b/src/app/shared/fd_config_json.c @@ -12,9 +12,7 @@ or knowingly skipped) before the constant is bumped. String keys of the user's own file are separately forced through the classification lists below. */ -/* topo.boot_identity and sign.identity_key_fd are derived startup state, - not operator configuration. */ -FD_STATIC_ASSERT( sizeof(fd_config_t)==25509048UL, update_fd_config_to_json_for_the_layout_change ); +FD_STATIC_ASSERT( sizeof(fd_config_t)==24460472UL, update_fd_config_to_json_for_the_layout_change ); #define REDACTED "[redacted]" diff --git a/src/app/shared/test_config_json.c b/src/app/shared/test_config_json.c index a34f3b3d8d0..12048d1fdb5 100644 --- a/src/app/shared/test_config_json.c +++ b/src/app/shared/test_config_json.c @@ -139,15 +139,15 @@ main( int argc, FD_TEST( !strstr( json, "hunter2" ) ); FD_TEST( !strstr( json, "example.com" ) ); - /* failover settings are reported even when disabled */ - FD_TEST( strstr( json, "\"failover\":{\"enabled\":false,\"listen_address\":\"[redacted]\",\"listen_port\":8010}" ) ); + /* failover settings are reported, failover is on by default */ + FD_TEST( strstr( json, "\"failover\":{\"enabled\":true,\"listen_address\":\"[redacted]\",\"listen_port\":8010}" ) ); /* the listen address is redacted, the rest is reported */ - config->firedancer.failover.enabled = 1; + config->firedancer.failover.enabled = 0; strcpy( config->firedancer.failover.listen_address, "10.9.8.7" ); FD_TEST( fd_config_to_json( config, json, sizeof(json) ) ); FD_TEST( !strstr( json, "10.9.8.7" ) ); - FD_TEST( strstr( json, "\"failover\":{\"enabled\":true,\"listen_address\":\"[redacted]\",\"listen_port\":8010}" ) ); + FD_TEST( strstr( json, "\"failover\":{\"enabled\":false,\"listen_address\":\"[redacted]\",\"listen_port\":8010}" ) ); /* user override toml renders with keyword redaction */ static char const user_toml[] = diff --git a/src/app/shared/test_config_parse.c b/src/app/shared/test_config_parse.c index d917fbc0379..2ab692ee94f 100644 --- a/src/app/shared/test_config_parse.c +++ b/src/app/shared/test_config_parse.c @@ -239,7 +239,7 @@ main( int argc, FD_TEST( failover_is_valid( config, 1, "10.0.0.1", 8010, "vote.json" ) ); FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 0, "vote.json" ) ); FD_TEST( !failover_is_valid( config, 1, "localhost", 8010, "vote.json" ) ); - FD_TEST( !failover_is_valid( config, 1, "0.0.0.0", 8010, "" ) ); + FD_TEST( failover_is_valid( config, 1, "0.0.0.0", 8010, "" ) ); /* inactive without a vote account */ FD_TEST( failover_is_valid( config, 0, "", 0, "" ) ); /* Ensure we can selectively override a field */ diff --git a/src/choreo/votor/ag_votor.c b/src/choreo/votor/ag_votor.c index cc724f69ad5..be912cc13bb 100644 --- a/src/choreo/votor/ag_votor.c +++ b/src/choreo/votor/ag_votor.c @@ -646,6 +646,14 @@ ag_votor_mark_unsent( ag_votor_t * self, if( FD_UNLIKELY( vote->kind==AG_VOTE_KIND_NOTAR ) ) { state->voted_notar = 0; fd_memset( state->voted_notar_hash, 0, sizeof(ag_block_hash_t) ); + /* A final queued behind this notar must not go out, no final follows + a discarded vote. The queue is rebuilt without it. */ + ulong cnt = vote_events_cnt( self->vote_events ); + for( ulong i=0UL; ivote_events ); + if( FD_UNLIKELY( event.vote.kind==AG_VOTE_KIND_FINAL && ag_vote_slot( &event.vote )==slot ) ) continue; + vote_events_push( self->vote_events, event ); + } } } diff --git a/src/disco/events/generated/fd_event_gen.h b/src/disco/events/generated/fd_event_gen.h index 798780a21c5..683297c8a9e 100644 --- a/src/disco/events/generated/fd_event_gen.h +++ b/src/disco/events/generated/fd_event_gen.h @@ -517,7 +517,7 @@ typedef struct fd_event_snapshot_created fd_event_snapshot_created_t; #define FD_EVENT_ADMIN_COMMAND_TYPE_ADD_AUTHORIZED_VOTER (3) /* Add an authorized voter key. */ #define FD_EVENT_ADMIN_COMMAND_TYPE_REMOVE_ALL_AUTHORIZED_VOTERS (4) /* Remove every authorized voter key. */ #define FD_EVENT_ADMIN_COMMAND_TYPE_SNAPSHOT_CREATE (5) /* Request creation of a snapshot. */ -#define FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE (6) /* Promote the running standby validator to active. */ +#define FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE (6) /* Take over the identity specified by the given keypair from the machine currently running it. */ #define FD_EVENT_ADMIN_COMMAND_TYPE_FAILOVER_STATUS (7) /* Read the running validator's failover state. */ /* The result of processing the admin command. */ diff --git a/src/disco/events/schema/admin_command.json b/src/disco/events/schema/admin_command.json index f6ffe9e672d..4c0225fcdab 100644 --- a/src/disco/events/schema/admin_command.json +++ b/src/disco/events/schema/admin_command.json @@ -12,7 +12,7 @@ "add_authorized_voter": { "description": "Add an authorized voter key." }, "remove_all_authorized_voters": { "description": "Remove every authorized voter key." }, "snapshot_create": { "description": "Request creation of a snapshot." }, - "failover_promote": { "description": "Promote the running standby validator to active." }, + "failover_promote": { "description": "Take over the identity specified by the given keypair from the machine currently running it." }, "failover_status": { "description": "Read the running validator's failover state." } } }, diff --git a/src/disco/events/schema/events.proto b/src/disco/events/schema/events.proto index d1c204b265c..1009ee3950d 100644 --- a/src/disco/events/schema/events.proto +++ b/src/disco/events/schema/events.proto @@ -405,7 +405,7 @@ enum AdminCommandType { ADMIN_COMMAND_TYPE_ADD_AUTHORIZED_VOTER = 3; // Add an authorized voter key. ADMIN_COMMAND_TYPE_REMOVE_ALL_AUTHORIZED_VOTERS = 4; // Remove every authorized voter key. ADMIN_COMMAND_TYPE_SNAPSHOT_CREATE = 5; // Request creation of a snapshot. - ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE = 6; // Promote the running standby validator to active. + ADMIN_COMMAND_TYPE_FAILOVER_PROMOTE = 6; // Take over the identity specified by the given keypair from the machine currently running it. ADMIN_COMMAND_TYPE_FAILOVER_STATUS = 7; // Read the running validator's failover state. } diff --git a/src/disco/keyguard/fd_keyload.c b/src/disco/keyguard/fd_keyload.c index 2a9cd05fc3c..230c9d18aa8 100644 --- a/src/disco/keyguard/fd_keyload.c +++ b/src/disco/keyguard/fd_keyload.c @@ -11,14 +11,14 @@ #include #include -static FD_TL uchar boot_public_key[ 32 ]; -static FD_TL int boot_key_fd = -1; +static FD_TL uchar junk_public_key[ 32 ]; +static FD_TL int junk_key_fd = -1; void -fd_keyload_set_boot_identity( uchar const public_key[ 32 ], +fd_keyload_set_failover_junk( uchar const public_key[ 32 ], int key_fd ) { - fd_memcpy( boot_public_key, public_key, 32UL ); - boot_key_fd = key_fd; + fd_memcpy( junk_public_key, public_key, 32UL ); + junk_key_fd = key_fd; } uchar * FD_FN_SENSITIVE @@ -92,28 +92,28 @@ read_key( char const * key_path, } static void FD_FN_SENSITIVE -read_boot_key( uchar * key ) { - int key_fd = boot_key_fd; - boot_key_fd = -1; +read_junk_key( uchar * key ) { + int key_fd = junk_key_fd; + junk_key_fd = -1; struct stat st; if( FD_UNLIKELY( fstat( key_fd, &st ) || !S_ISFIFO( st.st_mode ) ) ) - FD_LOG_ERR(( "the generated boot identity requires a startup pipe" )); + FD_LOG_ERR(( "the failover junk key requires a startup pipe" )); for( ulong off=0UL; off<64UL; ) { long n = read( key_fd, key+off, 64UL-off ); if( FD_UNLIKELY( n==-1L && errno==EINTR ) ) continue; - if( FD_UNLIKELY( n<=0L ) ) FD_LOG_ERR(( "could not read the generated boot identity" )); + if( FD_UNLIKELY( n<=0L ) ) FD_LOG_ERR(( "could not read the failover junk key" )); off += (ulong)n; } long n; do n = read( key_fd, key+64UL, 1UL ); while( FD_UNLIKELY( n==-1L && errno==EINTR ) ); - if( FD_UNLIKELY( n ) ) FD_LOG_ERR(( "unexpected data after the generated boot identity" )); - if( FD_UNLIKELY( close( key_fd ) ) ) FD_LOG_ERR(( "closing the boot identity pipe failed (%i-%s)", errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( n ) ) FD_LOG_ERR(( "unexpected data after the failover junk key" )); + if( FD_UNLIKELY( close( key_fd ) ) ) FD_LOG_ERR(( "closing the failover junk key pipe failed (%i-%s)", errno, fd_io_strerror( errno ) )); fd_sha512_t * sha = fd_sha512_join( fd_sha512_new( key+FD_SHA512_ALIGN ) ); FD_TEST( sha ); fd_ed25519_public_from_private( key+64UL, key, sha ); if( FD_UNLIKELY( !fd_memeq( key+32UL, key+64UL, 32UL ) || - !fd_memeq( key+32UL, boot_public_key, 32UL ) ) ) - FD_LOG_ERR(( "the generated boot identity does not match its public key" )); + !fd_memeq( key+32UL, junk_public_key, 32UL ) ) ) + FD_LOG_ERR(( "the failover junk key does not match its public key" )); fd_memzero_explicit( key+64UL, 4096UL-64UL ); } @@ -130,22 +130,27 @@ fd_keyload_load( char const * key_path, uchar * key_page = fd_keyload_alloc_protected_pages( 1UL, 2UL ); if( FD_UNLIKELY( !strcmp( key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) ) { - if( FD_UNLIKELY( fd_memeq( boot_public_key, (uchar[ 32 ]){0}, 32UL ) ) ) FD_LOG_ERR(( "missing generated boot identity" )); - if( public_key_only ) fd_memcpy( key_page+32UL, boot_public_key, 32UL ); - else read_boot_key( key_page ); + if( FD_UNLIKELY( fd_memeq( junk_public_key, (uchar[ 32 ]){0}, 32UL ) ) ) FD_LOG_ERR(( "missing failover junk key" )); + if( public_key_only ) fd_memcpy( key_page+32UL, junk_public_key, 32UL ); + else read_junk_key( key_page ); } else { read_key( key_path, key_page ); } if( public_key_only ) fd_memzero_explicit( key_page, 32UL ); - return fd_keyload_mprotect_ro( public_key_only ? key_page+32UL : key_page, public_key_only ); + /* Make the key page read-only now that the key has been loaded. */ + if( FD_UNLIKELY( mprotect( key_page, 4096UL, PROT_READ ) ) ) + FD_LOG_ERR(( "mprotect failed (%i-%s)", errno, fd_io_strerror( errno ) )); + + if( public_key_only ) return key_page+32UL; + else return key_page; } uchar const * FD_FN_SENSITIVE fd_keyload_generate( void ) { uchar * key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); - if( FD_UNLIKELY( !fd_rng_secure( key, 32UL ) ) ) FD_LOG_ERR(( "could not generate the boot identity" )); + if( FD_UNLIKELY( !fd_rng_secure( key, 32UL ) ) ) FD_LOG_ERR(( "could not generate a keypair" )); fd_sha512_t * sha = fd_sha512_join( fd_sha512_new( key+FD_SHA512_ALIGN ) ); FD_TEST( sha ); fd_ed25519_public_from_private( key+32UL, key, sha ); @@ -176,7 +181,7 @@ void FD_FN_SENSITIVE fd_keyload_unload( uchar const * key, int public_key_only ) { void * key_page = public_key_only ? (uchar *)key-32UL : (uchar *)key; - ulong sz = (2UL*2UL+1UL)*4096UL; + ulong sz = (2UL*1UL+2UL)*4096UL; if( FD_UNLIKELY( mprotect( key_page, 4096UL, PROT_READ | PROT_WRITE ) ) ) FD_LOG_ERR(( "mprotect failed (%i-%s)", errno, fd_io_strerror( errno ) )); diff --git a/src/disco/keyguard/fd_keyload.h b/src/disco/keyguard/fd_keyload.h index c443679895b..fd41ebd9d3a 100644 --- a/src/disco/keyguard/fd_keyload.h +++ b/src/disco/keyguard/fd_keyload.h @@ -3,7 +3,7 @@ #include "../fd_disco_base.h" -/* Internal identity source selected by failover startup. */ +/* Internal identity source selected by failover after hand-off. */ #define FD_KEYLOAD_FAILOVER_JUNK "@failover-junk" /* fd_keyload_read() reads a JSON encoded keypair from the provided file @@ -34,7 +34,7 @@ fd_keyload_read( int key_fd, is the path on disk of the key file. The key file must exist, be readable, and have the form of a Solana keypair (64 element JSON array of bytes). FD_KEYLOAD_FAILOVER_JUNK instead selects the - generated identity installed by fd_keyload_set_boot_identity. + failover junk key installed by fd_keyload_set_failover_junk. If public_key_only is non-zero, zeros out the private part of the key and returns a pointer to the first byte (of 32) of the public part of the key in binary format. If @@ -56,12 +56,12 @@ uchar const * FD_FN_SENSITIVE fd_keyload_load( char const * key_path, int public_key_only ); -/* Set this tile thread's generated boot public key and private startup +/* Set this tile thread's failover junk public key and private startup pipe before loading FD_KEYLOAD_FAILOVER_JUNK. Only Sign gets a pipe, other tiles pass -1 and can load only the public key. */ void -fd_keyload_set_boot_identity( uchar const public_key[ 32 ], +fd_keyload_set_failover_junk( uchar const public_key[ 32 ], int key_fd ); /* Generate a fresh Ed25519 keypair in protected memory. The returned diff --git a/src/disco/keyguard/fd_keyswitch.h b/src/disco/keyguard/fd_keyswitch.h index 81f54715aca..6f77c666c29 100644 --- a/src/disco/keyguard/fd_keyswitch.h +++ b/src/disco/keyguard/fd_keyswitch.h @@ -39,7 +39,7 @@ struct __attribute__((aligned(FD_KEYSWITCH_ALIGN))) fd_keyswitch_private { ulong result; ulong param; uchar bytes[ 32768UL ]; - ulong operator; /* 1 when set-identity made this switch under failover, so votor can participate without failover adoption */ + ulong failover; /* 1 when the failover tile made this switch, so votor requires an adopted vote history before it participates, 0 for set-identity */ /* Padding to FD_KEYSWITCH_ALIGN here */ }; diff --git a/src/disco/keyguard/fd_sign_tile.c b/src/disco/keyguard/fd_sign_tile.c index 8738993d927..473bf1f8d7c 100644 --- a/src/disco/keyguard/fd_sign_tile.c +++ b/src/disco/keyguard/fd_sign_tile.c @@ -64,9 +64,9 @@ typedef struct { uchar * private_key; uchar const * failover_junk_key; /* read only copies loaded at boot under failover */ - uchar const * failover_staked_key; - uchar * failover_operator_key; /* a set-identity key other than junk and staked, zero while there is none */ - uchar const * failover_key; /* the failover identity, failover_staked_key or failover_operator_key */ + uchar const * failover_boot_key; + uchar * failover_target_key; /* a target key other than the junk and boot keys, zero while there is none */ + uchar const * failover_key; /* the failover target, either failover_boot_key or failover_target_key */ fd_keyguard_bls_key_t * bls_keys; @@ -136,43 +136,38 @@ failover_identity_allowed( fd_sign_ctx_t const * ctx, return 1; } -/* failover_housekeeping runs the keyswitches the failover way before +/* failover_keyswitch runs the keyswitches the failover way before the normal keyswitch code sees them. A failover switch selects the junk key - or the failover identity. set-identity also makes the new key the one - failover hands off, unless it is our junk key. The staked key keeps its - read only page from boot, another key is copied into its own, and the - normal switch installs it. Return 0 only when the normal switch must - install this request. Once acknowledged, the keyswitch belongs to - Admin and must not be read again in this housekeeping call. */ + or the failover target. A prepare loads a new target without installing + it. set-identity also makes the new key the target, unless it is our junk + key. The boot key keeps its read only page from boot, another key is + copied into its own, and the normal switch installs it. Return 0 only + when the normal switch must install this request. Once acknowledged, the + keyswitch belongs to Admin and must not be read again in this + housekeeping call. */ static int FD_FN_SENSITIVE -failover_housekeeping( fd_sign_ctx_t * ctx ) { +failover_keyswitch( fd_sign_ctx_t * ctx ) { if( FD_LIKELY( fd_keyswitch_state_query( ctx->keyswitch )!=FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) return 1; ulong param = fd_keyswitch_param_query( ctx->keyswitch ); if( FD_UNLIKELY( param==FD_KEYSWITCH_PARAM_IDENTITY_PREPARE ) ) { int allowed = failover_identity_allowed( ctx, ctx->keyswitch->bytes+32UL ) && !fd_memeq( ctx->keyswitch->bytes+32UL, ctx->failover_junk_key+32UL, 32UL ); if( FD_LIKELY( allowed ) ) { - fd_memcpy( ctx->failover_operator_key, ctx->keyswitch->bytes, 64UL ); - ctx->failover_key = ctx->failover_operator_key; + fd_memcpy( ctx->failover_target_key, ctx->keyswitch->bytes, 64UL ); + ctx->failover_key = ctx->failover_target_key; } fd_memzero_explicit( ctx->keyswitch->bytes, 64UL ); ctx->keyswitch->result = allowed ? FD_ADMINCTL_RESULT_SUCCESS : FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER; fd_keyswitch_state( ctx->keyswitch, allowed ? FD_KEYSWITCH_STATE_COMPLETED : FD_KEYSWITCH_STATE_FAILED ); return 1; } - /* Before a client prepares a target, set-identity retains its normal - behavior and does not select an identity for failover. */ - if( param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR && !ctx->failover_key ) return 0; - if( FD_UNLIKELY( param==FD_KEYSWITCH_PARAM_IDENTITY_CHECK || param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR ) ) { - uchar const * pubkey = ctx->keyswitch->bytes + (param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR ? 32UL : 0UL); - int allowed = failover_identity_allowed( ctx, pubkey ); - if( FD_UNLIKELY( !allowed || param==FD_KEYSWITCH_PARAM_IDENTITY_CHECK ) ) { - fd_memzero_explicit( ctx->keyswitch->bytes, 64UL ); - ctx->keyswitch->result = allowed ? FD_ADMINCTL_RESULT_SUCCESS : FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER; - fd_keyswitch_state( ctx->keyswitch, allowed ? FD_KEYSWITCH_STATE_COMPLETED : FD_KEYSWITCH_STATE_FAILED ); - return 1; - } + if( FD_UNLIKELY( param==FD_KEYSWITCH_PARAM_IDENTITY_CHECK ) ) { + int allowed = failover_identity_allowed( ctx, ctx->keyswitch->bytes ); + fd_memzero_explicit( ctx->keyswitch->bytes, 64UL ); + ctx->keyswitch->result = allowed ? FD_ADMINCTL_RESULT_SUCCESS : FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER; + fd_keyswitch_state( ctx->keyswitch, allowed ? FD_KEYSWITCH_STATE_COMPLETED : FD_KEYSWITCH_STATE_FAILED ); + return 1; } if( FD_UNLIKELY( param==FD_KEYSWITCH_PARAM_IDENTITY_PUBKEY ) ) { failover_select( ctx ); @@ -180,12 +175,12 @@ failover_housekeeping( fd_sign_ctx_t * ctx ) { } FD_TEST( param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR ); if( FD_UNLIKELY( !fd_memeq( ctx->keyswitch->bytes+32UL, ctx->failover_junk_key+32UL, 32UL ) ) ) { - if( ctx->failover_staked_key && fd_memeq( ctx->keyswitch->bytes+32UL, ctx->failover_staked_key+32UL, 32UL ) ) { - ctx->failover_key = ctx->failover_staked_key; - fd_memzero_explicit( ctx->failover_operator_key, 64UL ); + if( fd_memeq( ctx->keyswitch->bytes+32UL, ctx->failover_boot_key+32UL, 32UL ) ) { + ctx->failover_key = ctx->failover_boot_key; + fd_memzero_explicit( ctx->failover_target_key, 64UL ); } else { - fd_memcpy( ctx->failover_operator_key, ctx->keyswitch->bytes, 64UL ); - ctx->failover_key = ctx->failover_operator_key; + fd_memcpy( ctx->failover_target_key, ctx->keyswitch->bytes, 64UL ); + ctx->failover_key = ctx->failover_target_key; } } return 0; @@ -193,7 +188,7 @@ failover_housekeeping( fd_sign_ctx_t * ctx ) { static void FD_FN_SENSITIVE during_housekeeping_sensitive( fd_sign_ctx_t * ctx ) { - int identity_handled = ctx->failover_junk_key && failover_housekeeping( ctx ); + int identity_handled = ctx->failover_junk_key && failover_keyswitch( ctx ); if( FD_UNLIKELY( !identity_handled && fd_keyswitch_state_query( ctx->keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) { memcpy( ctx->private_key, ctx->keyswitch->bytes, 32UL ); @@ -210,13 +205,6 @@ during_housekeeping_sensitive( fd_sign_ctx_t * ctx ) { if( FD_UNLIKELY( ctx->av_keyswitch && fd_keyswitch_state_query( ctx->av_keyswitch )==FD_KEYSWITCH_STATE_SWITCH_PENDING ) ) { ulong param = fd_keyswitch_param_query( ctx->av_keyswitch ); if( FD_LIKELY( param==FD_KEYSWITCH_PARAM_AV_ADD ) ) { - if( FD_UNLIKELY( ctx->failover_key && fd_memeq( ctx->av_keyswitch->bytes+32UL, ctx->failover_key+32UL, 32UL ) ) ) { - FD_LOG_WARNING(( "keyswitch failed: the failover identity cannot be an authorized voter" )); - fd_memzero_explicit( ctx->av_keyswitch->bytes, 64UL ); - ctx->av_keyswitch->result = FD_ADMINCTL_RESULT_UNSUPPORTED; - fd_keyswitch_state( ctx->av_keyswitch, FD_KEYSWITCH_STATE_FAILED ); - return; - } if( FD_UNLIKELY( ctx->authorized_voters_cnt==FD_KEYGUARD_AUTH_VOTERS_MAX ) ) { FD_LOG_WARNING(( "keyswitch failed: maximum number of authorized voters reached" )); fd_memzero_explicit( ctx->av_keyswitch->bytes, 64UL ); @@ -462,32 +450,19 @@ static void FD_FN_SENSITIVE failover_load_keys( fd_sign_ctx_t * ctx, fd_topo_tile_t const * tile ) { ctx->failover_junk_key = NULL; - ctx->failover_staked_key = NULL; - ctx->failover_operator_key = NULL; + ctx->failover_boot_key = NULL; + ctx->failover_target_key = NULL; ctx->failover_key = NULL; if( FD_LIKELY( !tile->sign.failover_enabled ) ) return; - uchar * junk_key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); - fd_memcpy( junk_key, ctx->private_key, 64UL ); - ctx->failover_junk_key = fd_keyload_mprotect_ro( junk_key, /* public_key_only: */ 0 ); - ctx->failover_operator_key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); - if( !tile->sign.failover_staked_key_path[ 0 ] ) return; - ctx->failover_staked_key = fd_keyload_load( tile->sign.failover_staked_key_path, /* pubkey only: */ 0 ); - ctx->failover_key = ctx->failover_staked_key; - if( FD_UNLIKELY( fd_memeq( ctx->failover_junk_key+32UL, ctx->failover_staked_key+32UL, 32UL ) ) ) { - FD_LOG_ERR(( "the failover junk identity must differ from the [paths.identity_key] key, which is the staked identity under failover" )); - } - fd_sha512_t sha[ 1 ]; - FD_TEST( fd_sha512_join( fd_sha512_new( sha ) ) ); - uchar public_key[ 32 ]; - fd_ed25519_public_from_private( public_key, ctx->failover_staked_key, sha ); - if( FD_UNLIKELY( !fd_memeq( public_key, ctx->failover_staked_key+32UL, 32UL ) ) ) { - FD_LOG_ERR(( "the public key in [paths.identity_key] does not match its private key" )); - } - for( ulong i=0UL; iauthorized_voters_cnt; i++ ) { - if( FD_UNLIKELY( fd_memeq( ctx->authorized_voter_pubkeys[ i ], ctx->failover_staked_key+32UL, 32UL ) ) ) { - FD_LOG_ERR(( "the staked [paths.identity_key] key cannot be one of the [paths.authorized_voter_paths] under failover" )); - } + ctx->failover_junk_key = fd_keyload_load( FD_KEYLOAD_FAILOVER_JUNK, /* pubkey only: */ 0 ); + uchar * boot_key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); + fd_memcpy( boot_key, ctx->private_key, 64UL ); + ctx->failover_boot_key = fd_keyload_mprotect_ro( boot_key, /* public_key_only: */ 0 ); + ctx->failover_target_key = fd_keyload_alloc_protected_pages( 1UL, 2UL ); + ctx->failover_key = ctx->failover_boot_key; + if( FD_UNLIKELY( fd_memeq( ctx->failover_junk_key+32UL, ctx->failover_boot_key+32UL, 32UL ) ) ) { + FD_LOG_ERR(( "the failover junk key must differ from the [paths.identity_key] key" )); } } diff --git a/src/disco/keyguard/test_sign_tile.c b/src/disco/keyguard/test_sign_tile.c index 8d56fbf82e5..529850aa237 100644 --- a/src/disco/keyguard/test_sign_tile.c +++ b/src/disco/keyguard/test_sign_tile.c @@ -315,8 +315,8 @@ failover_setup( void ) { derive_fields( &ctx ); fd_memset( operator_key, 0, 64UL ); ctx.failover_junk_key = junk_key; - ctx.failover_staked_key = staked_key; - ctx.failover_operator_key = operator_key; + ctx.failover_boot_key = staked_key; + ctx.failover_target_key = operator_key; ctx.failover_key = staked_key; ctx.in[0].role = FD_KEYGUARD_ROLE_FAILOV; client.response_mtu = FD_ED25519_SIG_SZ; @@ -422,17 +422,17 @@ test_failover_keys( void ) { FD_TEST( failover_signed_by( cv, sizeof(cv), junk_key +32UL ) ); } - /* The staked key is the failover identity here, so it cannot become an authorized voter. */ + /* The failover identity can become an authorized voter as on main, + admin checks it when failover prepares or switches to it. */ fd_keyswitch_t voter_switch[1]; ctx.av_keyswitch = fd_keyswitch_join( fd_keyswitch_new( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); FD_TEST( ctx.av_keyswitch ); fd_memcpy( voter_switch->bytes, staked_key, 64UL ); voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; during_housekeeping( &ctx ); - FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_FAILED ); - FD_TEST( voter_switch->result==FD_ADMINCTL_RESULT_UNSUPPORTED ); - FD_TEST( !ctx.authorized_voters_cnt ); - for( ulong i=0UL; i<64UL; i++ ) FD_TEST( !voter_switch->bytes[ i ] ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_COMPLETED ); + FD_TEST( ctx.authorized_voters_cnt==1UL && fd_memeq( ctx.authorized_voter_pubkeys[ 0 ], staked_key+32UL, 32UL ) ); + for( ulong i=0UL; i<32UL; i++ ) FD_TEST( !voter_switch->bytes[ i ] ); FD_LOG_NOTICE(( "pass: failover keys" )); } @@ -452,7 +452,7 @@ operator_switch( fd_keyswitch_t * ks, } static void -test_failover_operator_key( void ) { +test_failover_target_key( void ) { failover_setup(); fd_keyswitch_t identity_switch[1]; ctx.keyswitch = fd_keyswitch_join( fd_keyswitch_new( identity_switch, FD_KEYSWITCH_STATE_UNLOCKED ) ); @@ -482,23 +482,23 @@ test_failover_operator_key( void ) { FD_TEST( fd_memeq( ctx.public_key, junk_key+32UL, 32UL ) && ctx.failover_key==operator_key && fd_memeq( operator_key, third, 64UL ) ); FD_TEST( failover_signed_by( proof, sizeof(proof), third+32UL ) ); - /* The failover identity cannot become an authorized voter, the - configured key may be one now. */ + /* Authorized voters change as on main, also to the failover + identity, admin checks it when failover switches to it. */ fd_keyswitch_t voter_switch[1]; ctx.av_keyswitch = fd_keyswitch_join( fd_keyswitch_new( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); FD_TEST( ctx.av_keyswitch ); fd_memcpy( voter_switch->bytes, third, 64UL ); voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; during_housekeeping( &ctx ); - FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_FAILED && !ctx.authorized_voters_cnt ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_COMPLETED && ctx.authorized_voters_cnt==1UL ); fd_keyswitch_state( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); fd_memcpy( voter_switch->bytes, staked_key, 64UL ); voter_switch->param = FD_KEYSWITCH_PARAM_AV_ADD; during_housekeeping( &ctx ); - FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_COMPLETED && ctx.authorized_voters_cnt==1UL ); + FD_TEST( fd_keyswitch_state_query( voter_switch )==FD_KEYSWITCH_STATE_COMPLETED && ctx.authorized_voters_cnt==2UL ); - /* After removing the voter, set-identity can select the staked key. - It goes back to its boot page and clears the operator page. */ + /* set-identity to the boot key puts it back on its boot page and + clears the operator page. */ voter_switch->param = FD_KEYSWITCH_PARAM_AV_CLEAR; fd_keyswitch_state( voter_switch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); during_housekeeping( &ctx ); @@ -546,7 +546,7 @@ child_exit_status( void (*fn)( void ) ) { } /* A switch to a key that is neither the junk key nor the failover - identity, and a proof over another junk key, stop the + identity (as well as a proof over another junk key) all stop the validator. */ static void @@ -575,7 +575,7 @@ main( int argc, test_bls_request_rejected(); test_identity_switch_without_failover(); test_failover_keys(); - test_failover_operator_key(); + test_failover_target_key(); test_failover_refusals(); FD_LOG_NOTICE(( "pass" )); fd_halt(); diff --git a/src/disco/topo/fd_topo.c b/src/disco/topo/fd_topo.c index 1076d03be0e..b1cbd59d098 100644 --- a/src/disco/topo/fd_topo.c +++ b/src/disco/topo/fd_topo.c @@ -223,8 +223,8 @@ FD_FN_PURE static ulong fd_topo_tile_extra_normal_pages( fd_topo_tile_t const * tile ) { ulong key_pages = 0UL; if( FD_UNLIKELY( !strcmp( tile->name, "sign" ) && tile->sign.failover_enabled ) ) { - /* The read only junk and staked key pages, the set-identity key - page, and their guards, next to the identity key. */ + /* The junk key, the read only copy of the boot key and the failover + target key, each with its guard pages, next to the identity key. */ key_pages += 15UL; } if( FD_UNLIKELY( tile->id_keyswitch_obj_id!=ULONG_MAX ) ) { @@ -238,9 +238,9 @@ fd_topo_tile_extra_normal_pages( fd_topo_tile_t const * tile ) { key_pages += 5UL; } if( FD_UNLIKELY( !strcmp( tile->name, "failov" ) ) ) { - /* The failover tile loads the junk and staked public keys at boot, - both at the same time. */ - key_pages += 10UL; + /* The failover tile loads one public key at a time at boot and + unloads it before the next. */ + key_pages += 5UL; } if( !strcmp( tile->name, "net" ) ) { diff --git a/src/disco/topo/fd_topo.h b/src/disco/topo/fd_topo.h index 52c7c601255..1ce4ccfd2d3 100644 --- a/src/disco/topo/fd_topo.h +++ b/src/disco/topo/fd_topo.h @@ -246,8 +246,7 @@ struct fd_topo_tile { } admin; struct { - char identity_key_path[ PATH_MAX ]; /* the boot identity, the junk key generated at boot with failover */ - char staked_key_path[ PATH_MAX ]; /* we read only its public half */ + char identity_key_path[ PATH_MAX ]; /* the boot identity, the initial handoff target */ char vote_account_path[ PATH_MAX ]; uint listen_addr; /* network byte order */ ushort port; @@ -414,9 +413,8 @@ struct fd_topo_tile { struct { char identity_key_path[ PATH_MAX ]; - int identity_key_fd; /* private startup pipe when the boot identity is generated */ + int failover_junk_fd; /* private startup pipe for the failover junk key, -1 without failover */ int failover_enabled; - char failover_staked_key_path[ PATH_MAX ]; ulong authorized_voter_paths_cnt; char authorized_voter_paths[ FD_KEYGUARD_AUTH_VOTERS_MAX ][ PATH_MAX ]; struct { @@ -629,7 +627,6 @@ struct fd_topo_tile { struct { char identity_key_path[ PATH_MAX ]; char vote_history_path[ PATH_MAX ]; - int failover_standby; uint target_gid; uint target_uid; ulong authorized_voter_paths_cnt; @@ -826,7 +823,7 @@ typedef struct { between them. */ struct fd_topo { char app_name[ 256UL ]; - uchar boot_identity[ 32 ]; /* generated public key, zero for a file-backed identity */ + uchar failover_junk[ 32 ]; /* public half of the junk key generated at boot for failover, zero without failover */ uchar props[ 32768UL ]; ulong sleep_obj_id; diff --git a/src/disco/topo/fd_topo_run.c b/src/disco/topo/fd_topo_run.c index 22f83088ea9..365f8b0b606 100644 --- a/src/disco/topo/fd_topo_run.c +++ b/src/disco/topo/fd_topo_run.c @@ -79,7 +79,7 @@ fd_topo_run_tile( fd_topo_t * topo, fd_topo_join_tile_workspaces( topo, tile, core_dump_level ); fd_topo_mlock_code(); - fd_keyload_set_boot_identity( topo->boot_identity, strcmp( tile->name, "sign" ) ? -1 : tile->sign.identity_key_fd ); + fd_keyload_set_failover_junk( topo->failover_junk, strcmp( tile->name, "sign" ) ? -1 : tile->sign.failover_junk_fd ); if( FD_UNLIKELY( tile_run->privileged_init ) ) tile_run->privileged_init( topo, tile ); diff --git a/src/discof/admin/fd_admin_tile.c b/src/discof/admin/fd_admin_tile.c index 3bbebccc813..97b39824e70 100644 --- a/src/discof/admin/fd_admin_tile.c +++ b/src/discof/admin/fd_admin_tile.c @@ -30,8 +30,8 @@ struct fd_admin_tile_ctx { int failover_switch; /* a failover identity switch is in progress */ ulong failover_epoch; /* failover epoch, bumped by every set-identity under failover */ int failover_pending; /* the failover tile has not been told about the last set-identity yet */ - uchar junk_pubkey[ 32 ]; /* the identity we booted with, the junk key under failover */ - uchar failover_pubkey[ 32 ]; /* the key failover hands off, [paths.identity_key] until set-identity installs another */ + uchar junk_pubkey[ 32 ]; /* the junk key generated at boot, a completed handover switches to it */ + uchar failover_pubkey[ 32 ]; /* the key failover hands off, the boot identity until set-identity or a prepare picks another */ /* Failover commands go to the failover tile over the bus, one at a time like snapshot creation. */ @@ -55,6 +55,7 @@ struct fd_admin_tile_ctx { char failover_args_json[ 96 ]; /* the parked command and its flags for the event */ ulong failover_args_json_len; char failover_cmd_cstr[ 48 ]; /* the parked command and its flags for the log, empty for status */ + int failover_retargeted; /* the parked command changed the failover target before it was forwarded */ }; typedef struct fd_admin_tile_ctx fd_admin_tile_ctx_t; @@ -120,13 +121,12 @@ privileged_init( fd_topo_t const * topo, fd_memcpy( ctx->identity_pubkey, fd_keyload_load( tile->admin.identity_key_path, /* pubkey only: */ 1 ), 32UL ); - /* With failover, we boot on the junk key, and the key failover hands off - starts as [paths.identity_key]. */ - ulong failov_idx = fd_topo_find_tile( topo, "failov", 0UL ); - if( FD_UNLIKELY( failov_idx!=ULONG_MAX ) ) { - fd_memcpy( ctx->junk_pubkey, ctx->identity_pubkey, 32UL ); - char const * path = topo->tiles[ failov_idx ].failov.staked_key_path; - if( path[ 0 ] ) fd_memcpy( ctx->failover_pubkey, fd_keyload_load( path, /* pubkey only: */ 1 ), 32UL ); + /* With failover, the boot identity is the key that failover hands + off, and a handoff switches to the junk key generated at boot. */ + if( FD_UNLIKELY( fd_topo_find_tile( topo, "failov", 0UL )!=ULONG_MAX ) ) { + FD_TEST( !fd_memeq( topo->failover_junk, (uchar[ 32 ]){0}, 32UL ) ); + fd_memcpy( ctx->junk_pubkey, topo->failover_junk, 32UL ); + fd_memcpy( ctx->failover_pubkey, ctx->identity_pubkey, 32UL ); } } @@ -140,8 +140,7 @@ unprivileged_init( fd_topo_t const * topo, ctx->poll_ticks = (long)(1e6*fd_tempo_tick_per_ns( NULL )); /* 1ms */ ctx->idle_poll_ticks = (long)((double)FD_SLEEP_PARK_CAP_NS*fd_tempo_tick_per_ns( NULL )); ctx->deadline_ticks = 0L; - ulong failov_idx = fd_topo_find_tile( topo, "failov", 0UL ); - ctx->failover_enabled = failov_idx!=ULONG_MAX && topo->tiles[ failov_idx ].failov.staked_key_path[ 0 ]; + ctx->failover_enabled = fd_topo_find_tile( topo, "failov", 0UL )!=ULONG_MAX; ctx->failover_slot_idx = ULONG_MAX; ctx->topo = topo; @@ -350,8 +349,8 @@ unprivileged_init( fd_topo_t const * topo, static fd_keyswitch_t * find_identity_keyswitch( fd_admin_tile_ctx_t * ctx, char const * tile_name ) { - fd_topo_t const * topo = ctx->topo; - ulong tile_idx = fd_topo_find_tile( topo, tile_name, 0UL ); + fd_topo_t const * topo = ctx->topo; + ulong tile_idx = fd_topo_find_tile( topo, tile_name, 0UL ); FD_TEST( tile_idx!=ULONG_MAX ); FD_TEST( topo->tiles[ tile_idx ].id_keyswitch_obj_id!=ULONG_MAX ); @@ -683,6 +682,16 @@ failover_identity_check_poll( fd_admin_tile_ctx_t * ctx ) { return result; } +/* Runs until every sign tile has responded. */ +static ulong +failover_identity_check( fd_admin_tile_ctx_t * ctx, + uchar const * identity ) { + failover_identity_check_begin( ctx, identity ); + ulong result; + while( (result = failover_identity_check_poll( ctx ))==ULONG_MAX ) FD_SPIN_PAUSE(); + return result; +} + /* With failover, set-identity still switches the same way. */ static void @@ -692,7 +701,7 @@ failover_operator_begin( fd_admin_tile_ctx_t * ctx, if( !fd_memeq( identity, ctx->junk_pubkey, 32UL ) ) fd_memcpy( ctx->failover_pubkey, identity, 32UL ); FD_BASE58_ENCODE_32_BYTES( identity, new_identity ); FD_BASE58_ENCODE_32_BYTES( ctx->failover_pubkey, failover_identity ); - FD_LOG_WARNING(( "set-identity switches to `%s` while failover is on, failover restarts on this machine for identity `%s`", new_identity, failover_identity )); + FD_LOG_NOTICE(( "set-identity switches to `%s`, failover restarts on this machine for identity `%s`", new_identity, failover_identity )); fd_topo_t const * topo = ctx->topo; for( ulong i=0UL; itile_cnt; i++ ) { @@ -700,7 +709,7 @@ failover_operator_begin( fd_admin_tile_ctx_t * ctx, fd_keyswitch_t * sign = fd_topo_obj_laddr( topo, topo->tiles[ i ].id_keyswitch_obj_id ); sign->param = FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR; } - find_identity_keyswitch( ctx, "votor" )->operator = 1UL; + find_identity_keyswitch( ctx, "votor" )->failover = 0UL; ctx->failover_pending = 1; } @@ -718,7 +727,7 @@ failover_operator_notify( fd_admin_tile_ctx_t * ctx, fd_memcpy( operator.failover_identity, ctx->failover_pubkey, 32UL ); fd_memcpy( msg->payload, &operator, sizeof(operator) ); ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); - fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_OPERATOR, ctx->failov_out_chunk, sizeof(*msg), 0UL, tspub, tspub ); + fd_stem_publish( stem, ctx->failov_out_idx, FD_FAILOVER_BUS_OPERATOR_ACTION, ctx->failov_out_chunk, sizeof(*msg), 0UL, tspub, tspub ); ctx->failov_out_chunk = fd_dcache_compact_next( ctx->failov_out_chunk, sizeof(*msg), ctx->failov_out_chunk0, ctx->failov_out_wmark ); ctx->failover_pending = 0; } @@ -775,29 +784,7 @@ set_identity( fd_admin_tile_ctx_t * ctx, return; } - /* With failover, a set-identity to the key already installed does - nothing, so failover is not restarted. */ - if( FD_UNLIKELY( ctx->failover_enabled && fd_memeq( req->keypair+32UL, ctx->identity_pubkey, 32UL ) ) ) { - FD_LOG_NOTICE(( "set-identity: `%s` is already the identity, nothing to do", old_identity )); - report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS ); - fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_SUCCESS ); - return; - } - - if( FD_UNLIKELY( ctx->failover_enabled ) ) { - failover_identity_check_begin( ctx, req->keypair+32UL ); - ulong result; - while( (result = failover_identity_check_poll( ctx ))==ULONG_MAX ) FD_SPIN_PAUSE(); - if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { - FD_LOG_WARNING(( "set-identity failed: the new failover identity is already an authorized voter" )); - report_admin_command_custom_result( &event, "authorized_voter" ); - fd_adminctl_complete( adminctl, slot_idx, result ); - return; - } - failover_operator_begin( ctx, req->keypair+32UL ); - } - - if( ctx->failov_out_idx!=ULONG_MAX ) find_identity_keyswitch( ctx, "votor" )->operator = 1UL; + if( FD_UNLIKELY( ctx->failover_enabled ) ) failover_operator_begin( ctx, req->keypair+32UL ); ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; ulong identity_outset = (ulong)fd_log_wallclock(); @@ -1071,9 +1058,6 @@ add_authorized_voter( fd_admin_tile_ctx_t * ctx, case FD_ADD_AUTHORIZED_VOTER_RESULT_DUPLICATE_AUTH_VOTER: report_admin_command_custom_result( &event, "duplicate_authorized_voter" ); break; - case FD_ADMINCTL_RESULT_UNSUPPORTED: - report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNSUPPORTED ); - break; default: FD_LOG_ERR(( "unexpected add-authorized-voter result %lu", result )); } @@ -1375,17 +1359,21 @@ failover_result_name( ulong result ) { switch( result ) { case FD_FAILOVER_CONTROL_RESULT_BUSY: return "busy"; case FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE: return "unresponsive"; - case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: return "bad_role"; + case FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER: return "wrong_holder"; case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: return "in_progress"; case FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY: return "peer_unready"; - case FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED: return "peer_unverified"; - case FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE: return "peer_active"; + case FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS: return "peer_holds"; case FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING: return "handoff_pending"; case FD_FAILOVER_CONTROL_RESULT_TAKEN: return "taken"; case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: return "staked_seen"; case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: return "no_final_history"; case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: return "replay_behind"; - case FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS: return "no_active_address"; + case FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS: return "no_holder_address"; + case FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT: return "key_conflict"; + case FD_FAILOVER_CONTROL_RESULT_INPUT_LOST: return "input_lost"; + case FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH: return "keypair_mismatch"; + case FD_SET_IDENTITY_RESULT_SAME_IDENTITY: return "same_identity"; + case FD_SET_IDENTITY_RESULT_AUTHORIZED_VOTER: return "authorized_voter"; default: return "refused"; } } @@ -1417,9 +1405,15 @@ failover_complete( fd_admin_tile_ctx_t * ctx, if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) { FD_LOG_NOTICE(( "`%s` accepted by the failover tile, follow it with `failover status`", ctx->failover_cmd_cstr )); } else if( FD_UNLIKELY( result!=FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE ) ) { - FD_LOG_WARNING(( "`%s` refused by the failover tile (%s), nothing was done", ctx->failover_cmd_cstr, failover_result_name( result ) )); + if( FD_UNLIKELY( ctx->failover_retargeted ) ) { + FD_BASE58_ENCODE_32_BYTES( ctx->failover_pubkey, target ); + FD_LOG_WARNING(( "`%s` refused by the failover tile (%s); the failover target is now `%s`, nothing else was done", ctx->failover_cmd_cstr, failover_result_name( result ), target )); + } else { + FD_LOG_WARNING(( "`%s` refused by the failover tile (%s), nothing was done", ctx->failover_cmd_cstr, failover_result_name( result ) )); + } } } + ctx->failover_retargeted = 0; ctx->failover_slot_idx = ULONG_MAX; ctx->failover_start_time = 0UL; ctx->failover_deadline = 0L; @@ -1427,9 +1421,7 @@ failover_complete( fd_admin_tile_ctx_t * ctx, ctx->failover_cmd_cstr[ 0 ] = '\0'; } -/* Load a client's first target without changing its installed identity. - Repeating the same target leaves the controller and its history intact. - Admin serializes this with set-identity and authorized-voter changes. */ +/* Load a target key without changing the installed identity. */ static ulong FD_FN_SENSITIVE failover_prepare( fd_admin_tile_ctx_t * ctx, uchar const * keypair ) { @@ -1437,13 +1429,8 @@ failover_prepare( fd_admin_tile_ctx_t * ctx, fd_ed25519_public_from_private( pubkey, keypair, ctx->sha512 ); if( FD_UNLIKELY( !fd_memeq( pubkey, keypair+32UL, 32UL ) ) ) return FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH; if( FD_UNLIKELY( fd_memeq( pubkey, ctx->junk_pubkey, 32UL ) ) ) return FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT; - if( ctx->failover_enabled ) return fd_memeq( pubkey, ctx->failover_pubkey, 32UL ) - ? FD_ADMINCTL_RESULT_SUCCESS : FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT; - if( FD_UNLIKELY( !fd_memeq( ctx->identity_pubkey, ctx->junk_pubkey, 32UL ) ) ) return FD_FAILOVER_CONTROL_RESULT_NOT_BOOT_IDENTITY; - - failover_identity_check_begin( ctx, pubkey ); - ulong result; - while( (result = failover_identity_check_poll( ctx ))==ULONG_MAX ) FD_SPIN_PAUSE(); + if( fd_memeq( pubkey, ctx->failover_pubkey, 32UL ) ) return FD_ADMINCTL_RESULT_SUCCESS; + ulong result = failover_identity_check( ctx, pubkey ); if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) return result; for( ulong i=0UL; itopo->tile_cnt; i++ ) { @@ -1458,9 +1445,9 @@ failover_prepare( fd_admin_tile_ctx_t * ctx, /* Preflight checked every sign tile before any target was loaded. */ FD_TEST( result==FD_ADMINCTL_RESULT_SUCCESS ); fd_memcpy( ctx->failover_pubkey, pubkey, 32UL ); - ctx->failover_enabled = 1; ctx->failover_epoch++; - ctx->failover_pending = 1; + ctx->failover_pending = 1; + ctx->failover_retargeted = 1; return FD_ADMINCTL_RESULT_SUCCESS; } @@ -1510,7 +1497,6 @@ failover_request( fd_admin_tile_ctx_t * ctx, else if( (req.cmd==FD_ADMINCTL_FAILOVER_CMD_PROMOTE)!=force ) invalid = "--force goes with promote, and a promote without it is sent as a handoff"; else if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_STATUS && req.flags ) invalid = "failover status takes no flags"; else if( req.cmd!=FD_ADMINCTL_FAILOVER_CMD_HANDOFF && (req.addr || req.port) ) invalid = "an address or port goes only with a handoff"; - else if( req.reserved[ 0 ] || req.reserved[ 1 ] ) invalid = "reserved bytes are set"; if( FD_UNLIKELY( invalid ) ) { FD_LOG_WARNING(( "refusing adminctl failover cmd %lu flags %lu, %s", req.cmd, req.flags, invalid )); report_admin_command( &event, FD_EVENT_ADMIN_COMMAND_RESULT_UNKNOWN_COMMAND ); @@ -1558,9 +1544,11 @@ failover_request( fd_admin_tile_ctx_t * ctx, return; } fd_adminctl_failover_prepare_t * payload = data; - ulong result = failover_prepare( ctx, payload->keypair ); + ctx->failover_retargeted = 0; + ulong result = failover_prepare( ctx, payload->keypair ); fd_memzero_explicit( payload->keypair, sizeof(payload->keypair) ); if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + FD_LOG_WARNING(( "`%s` refused before it reached the failover tile (%s), nothing was done", cmd_cstr, failover_result_name( result ) )); report_admin_command_custom_result( &event, "target_refused" ); fd_adminctl_complete( adminctl, slot_idx, result ); return; @@ -1635,6 +1623,11 @@ failover_switch_request( fd_admin_tile_ctx_t * ctx, response.operator.epoch = ctx->failover_epoch; fd_memcpy( response.operator.identity, ctx->identity_pubkey, 32UL ); fd_memcpy( response.operator.failover_identity, ctx->failover_pubkey, 32UL ); + } else if( FD_UNLIKELY( ctx->failover_enabled && !fd_memeq( req.identity, ctx->junk_pubkey, 32UL ) && + failover_identity_check( ctx, req.identity )!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + FD_BASE58_ENCODE_32_BYTES( req.identity, target ); + FD_LOG_WARNING(( "refusing the failover tile's identity switch to `%s`, it is one of the authorized voters, remove it with remove-all-authorized-voters first", target )); + response.result = FD_FAILOVER_SWITCH_ERR_AUTHORIZED_VOTER; } else if( FD_LIKELY( ctx->failover_enabled ) ) { fd_event_admin_command_t event = prepare_admin_command( FD_EVENT_ADMIN_COMMAND_TYPE_SET_IDENTITY, NULL, 0UL ); FD_BASE58_ENCODE_32_BYTES( ctx->identity_pubkey, old_identity ); @@ -1648,7 +1641,7 @@ failover_switch_request( fd_admin_tile_ctx_t * ctx, fd_memset( keypair, 0, 32UL ); fd_memcpy( keypair+32UL, req.identity, 32UL ); - find_identity_keyswitch( ctx, "votor" )->operator = 0UL; /* votor enforces adoption before target participation */ + find_identity_keyswitch( ctx, "votor" )->failover = 1UL; /* votor enforces adoption before target participation */ ctx->failover_switch = 1; ulong state = FD_SET_IDENTITY_STATE_UNLOCKED; ulong identity_outset = (ulong)fd_log_wallclock(); @@ -1742,13 +1735,17 @@ after_credit( fd_admin_tile_ctx_t * ctx, break; case FD_ADMINCTL_CMD_FAILOVER_PROMOTE: case FD_ADMINCTL_CMD_FAILOVER_STATUS: - if( FD_UNLIKELY( ctx->failov_out_idx!=ULONG_MAX ) ) { - failover_request( ctx, stem, slot_idx, cmd_id, payload, payload_sz ); - *charge_busy = 1; - *opt_poll_in = 0; + if( FD_UNLIKELY( ctx->failov_out_idx==ULONG_MAX ) ) { + /* The command reads UNKNOWN_COMMAND as failover disabled. */ + FD_LOG_NOTICE(( "`%s` requested but failover is not enabled on this validator", + cmd_id==FD_ADMINCTL_CMD_FAILOVER_STATUS ? "failover status" : "failover promote" )); + fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ); break; } - __attribute__((fallthrough)); + failover_request( ctx, stem, slot_idx, cmd_id, payload, payload_sz ); + *charge_busy = 1; + *opt_poll_in = 0; + break; default: FD_LOG_WARNING(( "unexpected adminctl cmd %lu", cmd_id )); fd_adminctl_complete( adminctl, slot_idx, FD_ADMINCTL_RESULT_UNKNOWN_COMMAND ); diff --git a/src/discof/admin/fd_adminctl.h b/src/discof/admin/fd_adminctl.h index 47f25af930a..ae95028aa13 100644 --- a/src/discof/admin/fd_adminctl.h +++ b/src/discof/admin/fd_adminctl.h @@ -69,21 +69,18 @@ #define FD_FAILOVER_CONTROL_RESULT_BUSY (0x4001UL) /* another failover command is waiting on the failover tile */ #define FD_FAILOVER_CONTROL_RESULT_UNRESPONSIVE (0x4002UL) /* the failover tile did not respond in time */ -#define FD_FAILOVER_CONTROL_RESULT_BAD_ROLE (0x4003UL) /* promote on the active */ +#define FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER (0x4003UL) /* this machine already runs the identity, or the one it asked does not */ #define FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS (0x4004UL) /* a transition or key switch is running */ -#define FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY (0x4006UL) /* the bound peer cannot complete this handoff */ -#define FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE (0x4007UL) /* the authenticated peer holds the identity */ -#define FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING (0x4008UL) /* the peer has not responded to our handoff */ -#define FD_FAILOVER_CONTROL_RESULT_TAKEN (0x4009UL) /* the peer took our handoff, cleared once we hold the identity again or set-identity restarts failover */ -#define FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN (0x400AUL) /* gossip has a fresh contact info for the staked identity from another host */ -#define FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY (0x400CUL) /* active has no eligible final vote state, see its local voting diagnostics */ -#define FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS (0x400EUL) /* gossip has no address for the active */ -#define FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED (0x400FUL) /* no check found another holder, which does not prove the peer cannot sign */ -#define FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND (0x4010UL) /* this standby's replay is too far behind the active's last vote */ -#define FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED (0x4011UL) /* client has no target identity */ -#define FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT (0x4012UL) /* target differs from the prepared identity or is the boot identity */ -#define FD_FAILOVER_CONTROL_RESULT_NOT_BOOT_IDENTITY (0x4013UL) /* client must prepare while still on its boot identity */ -#define FD_FAILOVER_CONTROL_RESULT_INPUT_LOST (0x4014UL) /* failover input overrun, restart required */ +#define FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY (0x4005UL) /* the bound peer cannot complete this handoff */ +#define FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS (0x4006UL) /* the authenticated peer runs the identity */ +#define FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING (0x4007UL) /* the peer has not responded to our handoff */ +#define FD_FAILOVER_CONTROL_RESULT_TAKEN (0x4008UL) /* the peer took our handoff, cleared once we hold the identity again or set-identity restarts failover */ +#define FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN (0x4009UL) /* gossip has a fresh contact info for the staked identity from another host */ +#define FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY (0x400AUL) /* the machine running the identity has no eligible final vote state, see its local voting diagnostics */ +#define FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS (0x400BUL) /* gossip has no address for the machine running the identity */ +#define FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND (0x400CUL) /* this machine's replay is too far behind the last vote of the machine running the identity */ +#define FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT (0x400DUL) /* the keypair is the junk key */ +#define FD_FAILOVER_CONTROL_RESULT_INPUT_LOST (0x400EUL) /* failover input overrun, restart required */ struct fd_adminctl_add_auth_voter_v1 { ulong version; /* ==FD_ADMINCTL_ADD_AUTH_VOTER_PAYLOAD_VERSION */ @@ -141,7 +138,7 @@ fd_adminctl_failover_cmd_typed( ulong cmd ) { case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: return "failover promote"; case FD_ADMINCTL_FAILOVER_CMD_STATUS: return "failover status"; - default: return "unknown"; + default: return "unknown"; } } @@ -152,9 +149,8 @@ struct fd_adminctl_failover_req_v1 { ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ ulong cmd; /* FD_ADMINCTL_FAILOVER_CMD_* */ ulong flags; /* FD_ADMINCTL_FAILOVER_FLAG_* */ - uint addr; /* HANDOFF only, the active to dial in network byte order, 0 to find it in gossip */ - ushort port; /* HANDOFF only, the port to dial it on in host byte order, 0 for our own listen port */ - uchar reserved[ 2 ]; + uint addr; /* HANDOFF only, the machine running the identity to dial in network byte order, 0 to find it in gossip */ + ushort port; /* HANDOFF only, the port to dial it on in host byte order, 0 for the default port */ }; typedef struct fd_adminctl_failover_req_v1 fd_adminctl_failover_req_t; @@ -168,9 +164,8 @@ typedef struct fd_adminctl_failover_prepare_v1 fd_adminctl_failover_prepare_t; struct fd_adminctl_failover_control_resp_v1 { ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ - uchar role; /* FD_FAILOVER_ROLE_* after the command */ + uchar holds; /* this machine runs the identity after the command */ uchar action; /* controller action after the command */ - uchar reserved[ 6 ]; ulong handoff_id; /* the handoff an accepted promote opened or resumed, 0 none */ }; typedef struct fd_adminctl_failover_control_resp_v1 fd_adminctl_failover_control_resp_t; @@ -186,8 +181,8 @@ FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)==24UL, failover_co #define FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY (3UL) /* waiting for replay to reach the finality anchor */ #define FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT (4UL) /* waiting for votor to adopt */ #define FD_FAILOVER_ACTION_PROMOTE_SWITCH (5UL) /* waiting for the staked key to be installed */ -#define FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER (6UL) /* requesting the active's final history */ -#define FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT (7UL) /* waiting for the old active to record our response */ +#define FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER (6UL) /* requesting the final history from the machine running the identity */ +#define FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT (7UL) /* waiting for the machine that handed it over to record our response */ #define FD_FAILOVER_ACTION_DEMOTE_DRAIN (8UL) /* junk key installed, waiting for votor history to drain */ #define FD_FAILOVER_ACTION_CNT (9UL) @@ -199,39 +194,41 @@ FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_control_resp_t)==24UL, failover_co /* The outcome of the last handoff this machine requested or gave. */ #define FD_FAILOVER_HANDOFF_NONE (0UL) /* nothing requested or sent */ #define FD_FAILOVER_HANDOFF_PENDING (1UL) /* requested or sent, no answer yet */ -#define FD_FAILOVER_HANDOFF_TAKEN (2UL) /* the standby took the identity */ -#define FD_FAILOVER_HANDOFF_DECLINED (3UL) /* the standby did not take the identity, or the old active did not record that it did */ +#define FD_FAILOVER_HANDOFF_TAKEN (2UL) /* the requesting machine took the identity */ +#define FD_FAILOVER_HANDOFF_DECLINED (3UL) /* it did not take the identity, or the machine that handed it over did not record that it did */ #define FD_FAILOVER_HANDOFF_RESTARTED (4UL) /* the peer came back with a new boot_id */ #define FD_FAILOVER_HANDOFF_CANCELLED (5UL) /* `failover promote --force` stopped waiting for the peer */ -#define FD_FAILOVER_HANDOFF_NOT_ACTIVE (6UL) /* the machine we dialed was a standby */ +#define FD_FAILOVER_HANDOFF_NOT_HOLDER (6UL) /* the machine we dialed does not run the identity */ #define FD_FAILOVER_HANDOFF_CNT (7UL) /* `failover status`, provided by the failover tile. */ struct fd_adminctl_failover_status_resp_v1 { - ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ - uchar enabled; /* 1 accepts handoffs, 2 is an outgoing client only */ - uchar role; /* FD_FAILOVER_ROLE_* */ - uchar action; /* FD_FAILOVER_ACTION_* */ - uchar stuck; /* a transition failed or is overdue */ - uchar link_state; /* FD_FAILOVER_SESSION_* */ - uchar peer_role_valid; /* a peer is authenticated on the current session */ - uchar peer_role; /* FD_FAILOVER_ROLE_* from HELLO or the handoff result */ - uchar handoff_result; /* FD_FAILOVER_HANDOFF_* of our last handoff */ - ulong peer_boot_id; /* boot_id of the last peer we paired with, 0 none */ - uint peer_addr; /* the address we dial, from `failover promote --address` or the active's from gossip, 0 none */ - ushort peer_port; /* the failover port we dial it on, in host byte order */ - uchar promote_source; /* FD_FAILOVER_SOURCE_* `failover promote --force` would adopt now */ - uchar peer_addr_cmd; /* peer_addr is from `failover promote --address`, else it came from gossip */ - ulong handoff_id; /* id of our last handoff, 0 none */ - ulong promote_floor; /* coverage floor a promote has to reach, ULONG_MAX none */ - ulong promote_result; /* what the takeover checks report without --force now, PEER_UNVERIFIED when they find no other holder */ - uchar mode; /* FD_FAILOVER_MODE_* the validator runs */ - uchar request_paused; /* our handoff request stopped at its deadline, `failover promote` resumes it */ - uchar reserved[ 6 ]; + ulong version; /* ==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION */ + uchar enabled; /* 1 when failover runs */ + uchar holds; /* this machine runs the identity */ + uchar action; /* FD_FAILOVER_ACTION_* */ + uchar stuck; /* a transition failed or is overdue */ + uchar link_state; /* FD_FAILOVER_SESSION_* */ + uchar peer_holds_valid; /* a peer is authenticated on the current session */ + uchar peer_holds; /* the peer runs the identity, from HELLO or the handoff result */ + uchar handoff_result; /* FD_FAILOVER_HANDOFF_* of our last handoff */ + ulong peer_boot_id; /* boot_id of the last peer we paired with, 0 none */ + uint peer_addr; /* the address we dial, from `failover promote --address` or from gossip, 0 none */ + ushort peer_port; /* the failover port we dial it on, in host byte order */ + uchar promote_source; /* FD_FAILOVER_SOURCE_* `failover promote --force` would adopt now */ + uchar peer_addr_cmd; /* peer_addr is from `failover promote --address`, else it came from gossip */ + ulong handoff_id; /* id of our last handoff, 0 none */ + ulong promote_floor; /* coverage floor a promote has to reach, ULONG_MAX none */ + ulong promote_result; /* what the takeover checks report without --force now, SUCCESS when they find no other holder */ + uchar mode; /* FD_FAILOVER_MODE_* the validator runs */ + uchar request_paused; /* our handoff request stopped at its deadline, `failover promote` resumes it */ + uchar target[ 32 ]; /* the identity failover hands off, from [paths.identity_key], set-identity or `failover promote ` */ + uchar installed[ 32 ]; /* the identity this machine signs with now */ + uchar junk[ 32 ]; /* this boot's junk key, installed when the identity is handed off */ }; typedef struct fd_adminctl_failover_status_resp_v1 fd_adminctl_failover_status_resp_t; -FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_status_resp_t)==64UL, failover_status_resp_v1_layout ); +FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_status_resp_t)==160UL, failover_status_resp_v1_layout ); FD_STATIC_ASSERT( sizeof(fd_adminctl_failover_status_resp_t)<=FD_ADMINCTL_PAYLOAD_MAX, failover_status_resp_fits ); /* fd_adminctl_failover_status_resp_init clears a status response and diff --git a/src/discof/admin/test_admin_tile.c b/src/discof/admin/test_admin_tile.c index 698a041f6cd..86e50a423f3 100644 --- a/src/discof/admin/test_admin_tile.c +++ b/src/discof/admin/test_admin_tile.c @@ -226,7 +226,7 @@ bus_setup( ulong credits ) { static fd_failover_operator_t bus_operator( void ) { - FD_TEST( bus_seq && bus_mcache[ bus_seq-1UL ].sig==FD_FAILOVER_BUS_OPERATOR ); + FD_TEST( bus_seq && bus_mcache[ bus_seq-1UL ].sig==FD_FAILOVER_BUS_OPERATOR_ACTION ); fd_failover_operator_t operator; fd_memcpy( &operator, ((fd_failover_bus_msg_t const *)bus_out)->payload, sizeof(operator) ); return operator; @@ -276,7 +276,7 @@ test_failover_command( ulong cmd_id, fd_adminctl_failover_req_t forwarded; fd_memcpy( &forwarded, ((fd_failover_bus_msg_t const *)bus_out)->payload, sizeof(forwarded) ); FD_TEST( forwarded.cmd==payload_cmd && forwarded.flags==flags ); - result = i ? FD_FAILOVER_CONTROL_RESULT_BAD_ROLE : FD_ADMINCTL_RESULT_SUCCESS; + result = i ? FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER : FD_ADMINCTL_RESULT_SUCCESS; event_result = i ? FD_EVENT_ADMIN_COMMAND_RESULT_CUSTOM : FD_EVENT_ADMIN_COMMAND_RESULT_SUCCESS; ctx.failov_in.nonce = ctx.failover_nonce; ctx.failov_in.result = result; @@ -319,7 +319,7 @@ test_failover_identity( void ) { nothing goes to the failover tile until every tile switched. */ failover_operator_begin( &ctx, third ); FD_TEST( ctx.failover_epoch==1UL && fd_memeq( ctx.failover_pubkey, third, 32UL ) && ctx.failover_pending && !bus_seq ); - FD_TEST( sign_ks->param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR && votor_ks->operator==1UL ); + FD_TEST( sign_ks->param==FD_KEYSWITCH_PARAM_IDENTITY_KEYPAIR && votor_ks->failover==0UL ); fd_memcpy( ctx.identity_pubkey, third, 32UL ); failover_operator_notify( &ctx, bus_stem ); fd_failover_operator_t operator = bus_operator(); @@ -367,6 +367,33 @@ test_failover_identity( void ) { fd_memcpy( &answer, ((fd_failover_bus_msg_t const *)bus_out)->payload, sizeof(answer) ); FD_TEST( answer.result==FD_FAILOVER_SWITCH_ERR_STALE && answer.operator.epoch==2UL ); FD_TEST( fd_memeq( answer.operator.identity, ctx.junk_pubkey, 32UL ) && fd_memeq( answer.operator.failover_identity, third, 32UL ) ); + + /* Another key becomes the target, also while this machine runs the + current one, the CLI warns about that case. */ + uchar keypair[ 64 ]; fd_memset( keypair, 0x77, 32UL ); + fd_ed25519_public_from_private( keypair+32UL, keypair, ctx.sha512 ); + fd_memcpy( ctx.identity_pubkey, third, 32UL ); + ctx.failover_pending = 0; + FD_TEST( fd_topob_new( test_topo, "admin-test" ) ); + fd_topob_wksp( test_topo, "wksp" )->wksp = NULL; + (void)mock_tile( "votor" ); + FD_TEST( failover_prepare( &ctx, keypair )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx.failover_epoch==3UL && ctx.failover_pending && ctx.failover_retargeted ); + FD_TEST( fd_memeq( ctx.failover_pubkey, keypair+32UL, 32UL ) && fd_memeq( ctx.identity_pubkey, third, 32UL ) ); + + /* Same key is a no-op, a mismatched keypair and the junk key are refused. */ + uchar other[ 64 ]; fd_memset( other, 0x78, 32UL ); + fd_ed25519_public_from_private( other+32UL, other, ctx.sha512 ); + FD_TEST( failover_prepare( &ctx, other )==FD_ADMINCTL_RESULT_SUCCESS ); + FD_TEST( ctx.failover_epoch==4UL && fd_memeq( ctx.failover_pubkey, other+32UL, 32UL ) ); + FD_TEST( failover_prepare( &ctx, other )==FD_ADMINCTL_RESULT_SUCCESS && ctx.failover_epoch==4UL ); + uchar bad[ 64 ]; fd_memcpy( bad, other, 64UL ); bad[ 32 ]^=1; + FD_TEST( failover_prepare( &ctx, bad )==FD_SET_IDENTITY_RESULT_KEYPAIR_MISMATCH ); + uchar junk[ 64 ]; fd_memset( junk, 0x79, 32UL ); + fd_ed25519_public_from_private( junk+32UL, junk, ctx.sha512 ); + fd_memcpy( ctx.junk_pubkey, junk+32UL, 32UL ); + FD_TEST( failover_prepare( &ctx, junk )==FD_FAILOVER_CONTROL_RESULT_KEY_CONFLICT ); + FD_TEST( ctx.failover_epoch==4UL && fd_memeq( ctx.failover_pubkey, other+32UL, 32UL ) ); ctx.topo = NULL; } diff --git a/src/discof/failover/fd_failover_bus.h b/src/discof/failover/fd_failover_bus.h index 4ed78ebe180..9cc4782180d 100644 --- a/src/discof/failover/fd_failover_bus.h +++ b/src/discof/failover/fd_failover_bus.h @@ -17,7 +17,7 @@ #define FD_FAILOVER_BUS_SWITCH_RESP (2UL) /* admin to failov, fd_failover_switch_resp_t */ #define FD_FAILOVER_BUS_REQUEST (3UL) /* admin to failov, fd_adminctl_failover_req_t */ #define FD_FAILOVER_BUS_RESPONSE (4UL) /* failov to admin, the requested command's response */ -#define FD_FAILOVER_BUS_OPERATOR (5UL) /* admin to failov, fd_failover_operator_t, set-identity ran */ +#define FD_FAILOVER_BUS_OPERATOR_ACTION (5UL) /* admin to failov, fd_failover_operator_t, set-identity ran */ /* Only the public key goes over the bus, and the sign tile selects its junk key or the failover identity. epoch is the failover epoch the @@ -30,14 +30,15 @@ struct fd_failover_switch_req { typedef struct fd_failover_switch_req fd_failover_switch_req_t; -#define FD_FAILOVER_SWITCH_OK (0UL) -#define FD_FAILOVER_SWITCH_ERR_DISABLED (1UL) /* failover is off here */ -#define FD_FAILOVER_SWITCH_ERR_STALE (2UL) /* set-identity ran since the request, nothing switched */ +#define FD_FAILOVER_SWITCH_OK (0UL) +#define FD_FAILOVER_SWITCH_ERR_DISABLED (1UL) /* failover is off here */ +#define FD_FAILOVER_SWITCH_ERR_STALE (2UL) /* set-identity ran since the request, nothing switched */ +#define FD_FAILOVER_SWITCH_ERR_AUTHORIZED_VOTER (3UL) /* the target is an authorized voter, nothing switched */ -/* set-identity ran, and every set-identity bumps the failover epoch. - identity is the public key it installed, and failover_identity is the - key failover hands off, which set-identity changes to identity unless - that is the junk key. */ +/* set-identity or failover promote ran, and each bumps the + failover epoch. identity is the installed public key, and + failover_identity is the key failover hands off, which set-identity + changes to identity unless that is the junk key. */ struct fd_failover_operator { ulong epoch; uchar identity[ 32 ]; diff --git a/src/discof/failover/fd_failover_channel.c b/src/discof/failover/fd_failover_channel.c index 75c7c16c2e3..3fc56584342 100644 --- a/src/discof/failover/fd_failover_channel.c +++ b/src/discof/failover/fd_failover_channel.c @@ -65,7 +65,6 @@ struct candidate { uchar transcript[ 32 ]; uchar client_proof[ 64 ]; fd_failover_tls_t tls; - fd_failover_wire_session_t wire; fd_failover_hello_t hello; uchar rx[ FD_FAILOVER_FRAME_MAX ]; uchar tx[ FD_FAILOVER_FRAME_MAX ]; @@ -88,6 +87,7 @@ struct known_source { }; struct fd_failover_channel { + /* Session */ ulong magic; /* FD_FAILOVER_CHANNEL_MAGIC once initialized */ int dial_peer; /* the session machine runs as the dialer */ ulong state; /* session state, stepped by fd_failover_session_step */ @@ -95,41 +95,57 @@ struct fd_failover_channel { ushort peer_port; /* the dial destination port */ int listen_fd; /* listening socket, -1 until it is open */ int paired_idx; /* the paired candidate, -1 while none is paired */ - ulong cursor; /* the next candidate to service, round robin */ - ulong proof_cursor; /* the next complete proof to verify, round robin */ + + /* Connection pool, serviced round robin */ + ulong cursor; /* the next candidate to service */ + ulong proof_cursor; /* the next complete proof to verify */ + + /* Credentials */ fd_failover_tls_ctx_t tls_ctx; /* TLS template every connection starts from */ fd_failover_hello_t self_hello; /* the HELLO we send */ fd_failover_hello_t peer_hello; /* the paired peer's HELLO, zero while unpaired */ - fd_failover_proof_signer_t proof_signer; - long verify_updated; - ulong verify_credit; + fd_failover_proof_signer_t proof_signer; /* signs our admission proofs in the sign tile */ + + /* Proof verification rate limits, credit is in nanoseconds of rate */ + long verify_updated; /* when verify_credit was last refilled */ + ulong verify_credit; /* proof checks per second across all addresses */ long priority_verify_updated; - ulong priority_verify_credit; - int verify_budget; + ulong priority_verify_credit; /* the same for addresses that authenticated before */ + int verify_budget; /* proof checks left this poll, at most one */ fd_sha512_t sha[ 1 ]; /* scratch for checking admission proofs */ + + /* Connections and the addresses they come from */ struct candidate candidates[ FD_FAILOVER_CHANNEL_CANDIDATE_MAX ]; /* connections that are handshaking or paired */ struct source sources[ SOURCE_CNT ]; /* accept rate limit per source address */ struct known_source known_sources[ KNOWN_SOURCE_CNT ]; /* authenticated addresses, expiring */ + + /* Accept and admission rate limits, credit is in nanoseconds of rate */ long rate_updated; /* when rate_credit was last refilled */ ulong rate_credit; /* accept syscall limit across all addresses */ long admit_updated; /* when admit_credit was last refilled */ ulong admit_credit; /* new candidate limit across all addresses */ - long reconnect_updated; - ulong reconnect_credit; + long reconnect_updated; /* when reconnect_credit was last refilled */ + ulong reconnect_credit; /* new candidate limit for addresses that authenticated before */ + + /* Timers */ long hello_timeout; /* how long a candidate has to pair */ long silence_timeout; /* a paired session silent for this long is dropped */ long last_rx; /* when the paired session last received a frame */ - long keepalive_at; + long keepalive_at; /* when the paired session sends its next KEEPALIVE */ long retry_at; /* when the dialer dials again */ long paired_at; /* when the current session paired */ long backoff; /* the current dial backoff, doubles up to backoff_max */ long backoff_min; /* the shortest dial backoff, also the jitter range */ long backoff_max; /* the longest dial backoff */ + + /* Log throttles, when each reason may be logged again */ long dial_log_at[ 3 ]; /* connect, early close, handshake timeout */ long tls_log_at[ 4 ]; /* transport, ALPN, authentication, other */ long hello_log_at[ 14 ]; /* one bounded bucket per HELLO refusal */ long loss_log_at[ 6 ]; /* transport, silence, TLS, protocol, framing, local encode */ long pair_log_at; /* when a pairing may be logged again */ + + /* The paired session */ int expected_close; /* RESULT was sent, peer may close normally */ ulong generation; /* identifies the current authenticated session */ }; @@ -177,7 +193,6 @@ close_candidate( struct candidate * c ) { c->fd = -1; c->dialed = 0; c->rx_used = c->tx_used = c->tx_sent = 0UL; - fd_memzero_explicit( &c->wire, sizeof(c->wire) ); } static void @@ -189,7 +204,7 @@ reset( fd_failover_channel_t * ch ) { fd_memset( &ch->peer_hello, 0, sizeof(ch->peer_hello) ); } -/* A standby may authenticate before its operator requests a handoff. */ +/* A machine may authenticate before its operator requests a handoff. */ static int want_dial( fd_failover_channel_t const * ch ) { return !!ch->peer_addr; @@ -267,10 +282,10 @@ fd_failover_channel_fini( fd_failover_channel_t * ch ) { } void -fd_failover_channel_set_role( fd_failover_channel_t * ch, - ulong role ) { - if( ch->self_hello.role==(uchar)role ) return; - ch->self_hello.role = (uchar)role; +fd_failover_channel_set_holds( fd_failover_channel_t * ch, + int holds ) { + if( ch->self_hello.holds==(uchar)!!holds ) return; + ch->self_hello.holds = (uchar)!!holds; for( ulong i=0UL; ipaired_idx ) close_candidate( &ch->candidates[i] ); if( ch->paired_idx<0 ) rest( ch ); @@ -304,7 +319,7 @@ fd_failover_channel_listen_port( fd_failover_channel_t const * ch ) { return fd_ushort_bswap( addr.sin_port ); } -/* Spreads a redial over up to one minimum backoff, so standbys that +/* Spreads a redial over up to one minimum backoff, so machines that dial each other do not keep colliding in step. */ static long jitter( fd_failover_channel_t const * ch ) { @@ -511,7 +526,6 @@ start( fd_failover_channel_t * ch, c->local_hello = ch->self_hello; c->rx_used = c->tx_used = c->tx_sent = 0UL; c->tls_started = 0; - fd_failover_wire_session_init( &c->wire ); return 0; } @@ -591,7 +605,7 @@ decode( fd_failover_channel_t * ch, struct candidate * c = &ch->candidates[idx]; uchar const * data; ulong frame_sz; - int err = fd_failover_wire_decode( &c->wire, c->rx, c->rx_used, type, &data, payload_sz, &frame_sz ); + int err = fd_failover_wire_decode( c->rx, c->rx_used, type, &data, payload_sz, &frame_sz ); if( err==FD_FAILOVER_WIRE_ERR_AGAIN ) return 0; if( err ) { lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_FRAME, "it sent a malformed frame, check that both machines run the same Firedancer version" ); return -1; } fd_memcpy( payload, data, *payload_sz ); @@ -607,12 +621,12 @@ hello_err_name( int err ) { switch( err ) { case HELLO_ERR_TLS_KEY: return "its junk key is not the key TLS authenticated, check who can reach the failover port"; case FD_FAILOVER_HELLO_ERR_VERSION: return "the protocol version differs, run the same Firedancer version on both machines"; - case FD_FAILOVER_HELLO_ERR_STAKED: return "the failover identity differs, both machines need the same handoff target, from [paths.identity_key] or their last `set-identity`"; + case FD_FAILOVER_HELLO_ERR_STAKED: return "the failover identity differs, both machines need the same handoff target, from [paths.identity_key], their last `set-identity` or `failover promote `"; case FD_FAILOVER_HELLO_ERR_VOTE_ACCT: return "the vote account differs, check that [paths.vote_account] is the same on both machines"; case FD_FAILOVER_HELLO_ERR_JUNK_EQ: return "its junk key is ours, check that the address is the other validator"; - case FD_FAILOVER_HELLO_ERR_JUNK_STAKE: return "a boot identity is the staked key, the standby must use a different identity"; - case FD_FAILOVER_HELLO_ERR_BOTH_ACT: return "both members are active, only one machine may run the staked identity, stop the validator that should not vote, a failover-enabled validator restarts as standby"; - case FD_FAILOVER_HELLO_ERR_ROLE: return "the role is unknown, run the same Firedancer version on both machines"; + case FD_FAILOVER_HELLO_ERR_JUNK_STAKE: return "a junk key is the identity being handed off, run the same Firedancer version on both machines"; + case FD_FAILOVER_HELLO_ERR_BOTH_HOLD: return "both machines run the identity, only one machine may run it, stop the validator that should not vote and set its [paths.identity_key] to a junk key before restarting it"; + case FD_FAILOVER_HELLO_ERR_HOLDS: return "it sent an unknown value, run the same Firedancer version on both machines"; case FD_FAILOVER_HELLO_ERR_BOOT_ID: return "the boot id is zero, run the same Firedancer version on both machines"; case FD_FAILOVER_HELLO_ERR_MODE: return "the consensus mode differs, run both machines in the same consensus mode"; case FD_FAILOVER_HELLO_ERR_PROOF: return "its admission proof is not signed by the failover identity, check that both machines hold the same keypair for it"; @@ -738,7 +752,7 @@ handshake_candidate( fd_failover_channel_t * ch, c->phase = PHASE_HELLO; /* The dialer sends first. On the listener we respond only to a HELLO that passed the checks below. */ - if( c->dialed ) c->tx_used = fd_failover_wire_encode( &c->wire, c->tx, FD_FAILOVER_MSG_HELLO, + if( c->dialed ) c->tx_used = fd_failover_wire_encode( c->tx, FD_FAILOVER_MSG_HELLO, (uchar const *)&c->local_hello, sizeof(c->local_hello) ); return 1; } @@ -775,7 +789,7 @@ read_candidate_hello( fd_failover_channel_t * ch, /* The peer is authenticated, finish any HELLO output before pairing. */ c->phase = PHASE_READY; if( !c->dialed ) { - c->tx_used = fd_failover_wire_encode( &c->wire, c->tx, FD_FAILOVER_MSG_HELLO, + c->tx_used = fd_failover_wire_encode( c->tx, FD_FAILOVER_MSG_HELLO, (uchar const *)&c->local_hello, sizeof(c->local_hello) ); if( flush( ch, idx, now, busy )!=0 ) return 0; /* Remain READY until output drains, unless the write fails. */ } @@ -812,9 +826,9 @@ pair_candidate( fd_failover_channel_t * ch, ch->pair_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); char member[ FD_BASE58_ENCODED_32_SZ ]; fd_base58_encode_32( c->hello.junk_pubkey, NULL, member ); - FD_LOG_NOTICE(( "paired with the failover peer at `" FD_IP4_ADDR_FMT "` via %s, authenticated member %s, role %s, boot id %016lx", + FD_LOG_NOTICE(( "paired with the failover peer at `" FD_IP4_ADDR_FMT "` via %s, authenticated member %s, %s, boot id %016lx", FD_IP4_ADDR_FMT_ARGS( c->address ), c->dialed ? "outbound dial" : "listener", member, - c->hello.role==FD_FAILOVER_ROLE_ACTIVE ? "active" : "standby", c->hello.boot_id )); + c->hello.holds ? "it runs the identity" : "it does not run the identity", c->hello.boot_id )); } } @@ -1179,7 +1193,7 @@ fd_failover_channel_send( fd_failover_channel_t * ch, ulong idx = (ulong)ch->paired_idx; struct candidate * c = &ch->candidates[idx]; if( c->tx_used ) return -1; - c->tx_used = fd_failover_wire_encode( &c->wire, c->tx, type, payload, sz ); + c->tx_used = fd_failover_wire_encode( c->tx, type, payload, sz ); if( !c->tx_used ) { lose( ch, idx, now, FD_FAILOVER_EV_LINK_LOST, LOSS_LOCAL_ENCODE, "we could not encode a frame for it" ); return -1; } int busy = 0; int rc = flush( ch, idx, now, &busy )<0 ? -1 : 0; diff --git a/src/discof/failover/fd_failover_channel.h b/src/discof/failover/fd_failover_channel.h index 7a5d1469970..7937fd6678e 100644 --- a/src/discof/failover/fd_failover_channel.h +++ b/src/discof/failover/fd_failover_channel.h @@ -99,11 +99,10 @@ fd_failover_channel_fini( fd_failover_channel_t * channel ); FD_FN_PURE ulong fd_failover_channel_pending( fd_failover_channel_t const * channel ); -/* Updates the role in the next HELLO and leaves a paired session up, so - an open handoff can finish after the roles change. */ +/* Updates whether the next HELLO says we run the handed off identity. */ void -fd_failover_channel_set_role( fd_failover_channel_t * channel, - ulong role ); +fd_failover_channel_set_holds( fd_failover_channel_t * channel, + int holds ); /* Drives the channel, now is the caller's clock in nanos. Sets *charge_busy when it made progress. Returns 1 with *out_type, diff --git a/src/discof/failover/fd_failover_proto.c b/src/discof/failover/fd_failover_proto.c index 5cfe48dae14..828d83f82a2 100644 --- a/src/discof/failover/fd_failover_proto.c +++ b/src/discof/failover/fd_failover_proto.c @@ -1,20 +1,17 @@ #include "fd_failover_proto.h" -#include "../../choreo/votor/ag_hist.h" #include "../../ballet/ed25519/fd_ed25519.h" #include "../../ballet/sha256/fd_sha256.h" #include "../../disco/keyguard/fd_keyguard.h" FD_STATIC_ASSERT( sizeof(FD_KEYGUARD_FAILOVER_CLIENT_DOMAIN)-1UL==FD_KEYGUARD_FAILOVER_DOMAIN_SZ, client_proof_domain ); FD_STATIC_ASSERT( sizeof(FD_KEYGUARD_FAILOVER_SERVER_DOMAIN)-1UL==FD_KEYGUARD_FAILOVER_DOMAIN_SZ, server_proof_domain ); -FD_STATIC_ASSERT( AG_HIST_SER_MAX<=FD_FAILOVER_ALPENGLOW_STATE_MAX, alpenglow_state_max ); int fd_failover_hello_check( fd_failover_hello_t const * self, fd_failover_hello_t const * peer ) { if( FD_UNLIKELY( self->version!=peer->version ) ) return FD_FAILOVER_HELLO_ERR_VERSION; - if( FD_UNLIKELY( self->role>FD_FAILOVER_ROLE_ACTIVE || - peer->role>FD_FAILOVER_ROLE_ACTIVE ) ) return FD_FAILOVER_HELLO_ERR_ROLE; + if( FD_UNLIKELY( self->holds>1 || peer->holds>1 ) ) return FD_FAILOVER_HELLO_ERR_HOLDS; if( FD_UNLIKELY( self->mode!=FD_FAILOVER_MODE_ALPENGLOW || self->mode!=peer->mode ) ) return FD_FAILOVER_HELLO_ERR_MODE; if( FD_UNLIKELY( !self->boot_id || !peer->boot_id ) ) return FD_FAILOVER_HELLO_ERR_BOOT_ID; @@ -23,8 +20,7 @@ fd_failover_hello_check( fd_failover_hello_t const * self, if( FD_UNLIKELY( fd_memeq( self->junk_pubkey, peer->junk_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_JUNK_EQ; if( FD_UNLIKELY( fd_memeq( self->junk_pubkey, self->staked_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_JUNK_STAKE; if( FD_UNLIKELY( fd_memeq( peer->junk_pubkey, peer->staked_pubkey, 32UL ) ) ) return FD_FAILOVER_HELLO_ERR_JUNK_STAKE; - if( FD_UNLIKELY( self->role==FD_FAILOVER_ROLE_ACTIVE && - peer->role==FD_FAILOVER_ROLE_ACTIVE ) ) return FD_FAILOVER_HELLO_ERR_BOTH_ACT; + if( FD_UNLIKELY( self->holds && peer->holds ) ) return FD_FAILOVER_HELLO_ERR_BOTH_HOLD; return FD_FAILOVER_HELLO_OK; } @@ -169,17 +165,6 @@ fd_failover_demoted_encode( uchar * out, return sizeof(demoted)+state_sz; } -/* Checks that the vote history decodes exactly and ends at the slot the - DEMOTED claims. */ -static int -alpenglow_state_check( uchar const * state, - ulong state_sz, - ulong vote_slot ) { - if( FD_UNLIKELY( state_sz>FD_FAILOVER_ALPENGLOW_STATE_MAX ) ) return 0; - ag_hist_t hist[1]; - return !ag_hist_de( state, state_sz, hist ) && ag_hist_tip( hist )==vote_slot; -} - int fd_failover_demoted_decode( fd_failover_demoted_t * out, uchar const * payload, @@ -191,13 +176,13 @@ fd_failover_demoted_decode( fd_failover_demoted_t * out, fd_memcpy( &demoted, payload, sizeof(demoted) ); ulong state_sz = payload_sz-sizeof(fd_failover_demoted_t); + /* The history itself is votor's to decode, the controller checks that + the adopted history ends at last_vote_slot. */ if( FD_UNLIKELY( (ulong)demoted.state_len!=state_sz || - !state_sz || + !state_sz || state_sz>FD_FAILOVER_ALPENGLOW_STATE_MAX || demoted.mode!=(uchar)FD_FAILOVER_MODE_ALPENGLOW || demoted.last_vote_slot==FD_FAILOVER_SLOT_NULL ) ) return 0; - if( FD_UNLIKELY( !alpenglow_state_check( payload+sizeof(fd_failover_demoted_t), state_sz, demoted.last_vote_slot ) ) ) return 0; - *out = demoted; return 1; } diff --git a/src/discof/failover/fd_failover_proto.h b/src/discof/failover/fd_failover_proto.h index 9921df7e44e..85c7b46efea 100644 --- a/src/discof/failover/fd_failover_proto.h +++ b/src/discof/failover/fd_failover_proto.h @@ -10,7 +10,7 @@ /* Protocol version */ #define FD_FAILOVER_VERSION (1U) -/* The port a standby dials unless `failover promote --port` gives +/* The port a handoff dials unless `failover promote --port` gives another. It is also the default [failover.listen_port]. */ #define FD_FAILOVER_DEFAULT_PORT ((ushort)8010) @@ -22,7 +22,7 @@ #define FD_FAILOVER_MSG_PROMOTE_REJECTED (4U) #define FD_FAILOVER_MSG_HANDOFF_RESULT (5U) #define FD_FAILOVER_MSG_KEEPALIVE (6U) -#define FD_FAILOVER_MSG_RESERVED (7U) +#define FD_FAILOVER_MSG_CNT (7U) /* Sentinel for a slot field with no value */ #define FD_FAILOVER_SLOT_NULL (ULONG_MAX) @@ -30,10 +30,6 @@ /* Consensus payload formats */ #define FD_FAILOVER_MODE_ALPENGLOW (1U) -/* Roles for each endpoint */ -#define FD_FAILOVER_ROLE_STANDBY (0UL) /* Run under the junk identity. */ -#define FD_FAILOVER_ROLE_ACTIVE (1UL) /* Run under the staked identity. */ - /* Session states for one authenticated TCP session per pair. The listener only ever shows LISTENING or PAIRED, its candidates are tracked per socket. The dialer walks BACKOFF, DIALING, HELLO and @@ -62,8 +58,8 @@ #define FD_FAILOVER_HELLO_ERR_VOTE_ACCT (3) #define FD_FAILOVER_HELLO_ERR_JUNK_EQ (4) #define FD_FAILOVER_HELLO_ERR_JUNK_STAKE (5) -#define FD_FAILOVER_HELLO_ERR_BOTH_ACT (6) -#define FD_FAILOVER_HELLO_ERR_ROLE (7) +#define FD_FAILOVER_HELLO_ERR_BOTH_HOLD (6) +#define FD_FAILOVER_HELLO_ERR_HOLDS (7) #define FD_FAILOVER_HELLO_ERR_BOOT_ID (8) #define FD_FAILOVER_HELLO_ERR_MODE (9) #define FD_FAILOVER_HELLO_ERR_PROOF (10) @@ -76,10 +72,10 @@ /* Wire protocol message bodies. Little endian, packed, fixed layout. */ struct __attribute__((packed)) fd_failover_hello { ushort version; /* FD_FAILOVER_VERSION */ - uchar junk_pubkey[ 32 ]; /* this host's junk identity */ - uchar staked_pubkey[ 32 ]; /* the active identity for this pair */ + uchar junk_pubkey[ 32 ]; /* this host's junk key, generated at boot */ + uchar staked_pubkey[ 32 ]; /* the identity this pair hands off */ uchar vote_account[ 32 ]; /* the vote account for this pair */ - uchar role; /* FD_FAILOVER_ROLE_* role at pairing */ + uchar holds; /* 1 when this host runs staked_pubkey at pairing, else 0 */ uchar mode; /* FD_FAILOVER_MODE_* consensus */ ulong boot_id; /* random nonzero value per boot to distinguish restarts */ uchar commit[ 20 ]; /* FD commit hash */ @@ -125,7 +121,7 @@ FD_STATIC_ASSERT( sizeof(fd_failover_admission_response_t)==66UL, admission_resp /* Sent by the demoter once it runs the junk key, the final history follows it. */ struct __attribute__((packed)) fd_failover_demoted { - ulong handoff_id; /* the handoff id from the standby's HANDOFF_REQUEST */ + ulong handoff_id; /* the handoff id from the HANDOFF_REQUEST */ ulong target_boot_id; /* peer boot_id the handoff is for */ ulong last_vote_slot; /* tip of the final history */ uchar mode; /* FD_FAILOVER_MODE_* encoding */ @@ -149,11 +145,12 @@ struct __attribute__((packed)) fd_failover_promote_rejected { typedef struct fd_failover_promote_rejected fd_failover_promote_rejected_t; FD_STATIC_ASSERT( sizeof(fd_failover_promote_rejected_t)==9UL, wire_layout ); -/* The standby asks the active it authenticated to hand over. */ +/* A machine asks the one it authenticated, which runs the identity, to + hand it over. */ struct __attribute__((packed)) fd_failover_handoff_request { ulong handoff_id; ulong target_boot_id; - ulong replay_slot; /* standby's replay slot, FD_FAILOVER_SLOT_NULL before replay starts */ + ulong replay_slot; /* replay slot of the machine asking for the identity, FD_FAILOVER_SLOT_NULL before replay starts */ }; typedef struct fd_failover_handoff_request fd_failover_handoff_request_t; FD_STATIC_ASSERT( sizeof(fd_failover_handoff_request_t)==24UL, wire_layout ); @@ -250,10 +247,9 @@ fd_failover_demoted_encode( uchar * out, uchar const * state, ulong state_sz ); -/* Validates a DEMOTED payload. The Alpenglow history must decode - exactly and end at last_vote_slot. Returns 1 on success, with the - history at payload+sizeof(fd_failover_demoted_t). Otherwise returns 0 - and leaves out unchanged. */ +/* Validates a DEMOTED header, not the history itself, which votor + decodes on adoption. Returns 1 on success, else 0 with out + unchanged. */ int fd_failover_demoted_decode( fd_failover_demoted_t * out, uchar const * payload, diff --git a/src/discof/failover/fd_failover_tile.c b/src/discof/failover/fd_failover_tile.c index 4e4385117e2..155d2c99029 100644 --- a/src/discof/failover/fd_failover_tile.c +++ b/src/discof/failover/fd_failover_tile.c @@ -19,7 +19,7 @@ #include "generated/fd_failover_tile_seccomp.h" /* The failov tile owns the sockets to the other failover machine, keeps - host networking for them and holds no private key. The sign tile + host networking for them and holds no private keys. The sign tile signs fresh admission proofs with the failover identity and our TLS handshakes with the junk key, and the admin tile does the identity switch and stays off the network. */ @@ -32,17 +32,19 @@ #define FD_FAILOVER_DEADLINE_SLOTS (64UL) /* The promotion's replay and adopt waits also end on our clock, one - second per deadline slot, so a standby whose replay froze stops. */ + second per deadline slot, so a machine whose replay froze eventually + stops. */ #define FD_FAILOVER_DEADLINE_SLOT_NANOS ((long)FD_FAILOVER_NANOS_PER_SECOND) -/* A staked contact info from another host newer than this means an - active is publishing, two gossip refreshes of 7.5s. */ +/* A contact info for the identity from another host newer than this + means that host runs it, two gossip refreshes of 7.5s. */ #define FD_FAILOVER_GOSSIP_FRESH_NANOS (15000000000L) +#define FD_FAILOVER_REQUEST_RETRIES (3UL) /* sends of one request before a missing final history ends it */ /* The controller actions, promotion sources and handoff results are in fd_adminctl.h, `failover status` reports them. */ -/* Which key a switch installs, CNT while none is in flight */ +/* Which key a switch installs, CNT if no switch is in flight */ #define FD_FAILOVER_SWITCH_KEY_JUNK (0UL) /* The junk identity. */ #define FD_FAILOVER_SWITCH_KEY_STAKED (1UL) /* The staked identity. */ #define FD_FAILOVER_SWITCH_KEY_CNT (2UL) /* No identity switch is pending. */ @@ -55,9 +57,9 @@ /* Our last response to a DEMOTED */ #define FD_FAILOVER_REPLY_NONE (0UL) /* No response yet. */ #define FD_FAILOVER_REPLY_KEPT (1UL) /* Kept, the same DEMOTED again gets it again. */ -#define FD_FAILOVER_REPLY_PENDING (2UL) /* Kept, not queued yet because the control slot was busy. */ +#define FD_FAILOVER_REPLY_PENDING (2UL) /* Kept, not queued yet (pending) because the control slot was busy. */ -/* A serialized vote history and its tip. */ +/* A serialized Alpenglow vote history and its tip. */ struct fd_failover_history { int valid; /* holds a history */ ulong tip; /* slot of its last vote */ @@ -84,20 +86,20 @@ peer_eq( fd_failover_peer_t const * bound, /* The session we last saw on the channel. */ struct fd_failover_session { - ulong state; /* last seen session state, for edge detection */ + ulong state; /* last seen session state */ ulong generation; /* catches a replacement paired during one poll */ ulong peer_boot_id; /* boot of the peer on this session */ - ulong peer_role; /* latest HELLO or accepted handoff seen on this session */ + int peer_holds; /* the peer runs the identity, from its HELLO or an accepted handoff on this session */ int close_after_send; /* hang up once the queued message is sent */ ulong close_handoff_id; /* the handoff that hang up is for, for the log */ }; typedef struct fd_failover_session fd_failover_session_t; -/* Our handoff request, sent by the standby. */ +/* Our handoff request, sent by the machine that wants the identity. */ struct fd_failover_request { ulong id; /* zero when none is open */ - fd_failover_peer_t peer; /* the active boot we authenticated for it */ + fd_failover_peer_t peer; /* the boot we authenticated for it, which runs the identity */ uint addr; /* the address we dial, a resume may replace it */ long until; /* when the request pauses, zero while paused */ int sent; /* sent on this session */ @@ -134,7 +136,7 @@ struct fd_failover_promotion { ulong handoff_id; /* and its handoff id */ ulong floor; /* coverage floor when it started */ int force; /* `failover promote --force`, we skip the peer checks and accept incomplete or empty history */ - int active_seen; /* an ACTIVE peer authenticated after promotion started */ + int holder_seen; /* a peer running the identity authenticated after promotion started */ long until; /* our clock when the replay and adopt waits give up, 0 until set */ fd_failover_history_t adopt; /* history or the bound sent to votor */ ulong adopt_id; /* id of the adoption we asked the votor for */ @@ -172,7 +174,7 @@ typedef struct fd_failover_tx fd_failover_tx_t; /* State of the identity switch we asked the admin tile for. It selects a key preloaded by the signing tile by its public key, and - success means the old key is gone from every tile's active identity. */ + success means the old key is gone from every tile's installed identity. */ struct fd_failover_id_switch { ulong request_id; /* nonce of our latest switch request */ ulong pending_key; /* FD_FAILOVER_SWITCH_KEY_*, or CNT when idle */ @@ -189,25 +191,25 @@ typedef struct fd_failover_id_switch fd_failover_id_switch_t; struct fd_failover_tile_ctx { fd_keyguard_client_t keyguard_client[ 1 ]; /* signs admission proofs and TLS */ - int client_only; /* no listener and no incoming handoffs */ - int input_lost; /* input overrun disables failover until restart */ - int read_pending; /* cleared only after STEM validates the fragment */ - int target_pending; /* waiting for Admin to prepare the client's target */ - + int input_lost; /* input overrun disables failover until restart */ + int read_pending; /* cleared only after STEM validates the fragment */ fd_failover_hello_t hello; /* the HELLO we send */ - ulong role; /* FD_FAILOVER_ROLE_* */ - uint cmd_addr; /* from `failover promote --address`, 0 when gossip finds the active */ + int holds; /* we run the target identity, hello.staked_pubkey */ + uchar installed[ 32 ]; /* the identity this machine signs with now, from the switches we asked for and set-identity */ + uint cmd_addr; /* from `failover promote --address`, 0 when gossip finds the identity */ ushort cmd_port; /* from `failover promote --port`, 0 for the default port */ fd_failover_channel_t * channel; /* the TLS channel to the peer */ fd_failover_session_t session; /* the session we last saw on the channel */ long poll_at; /* next idle socket poll */ - fd_failover_request_t request; /* our handoff request, as a standby */ + fd_failover_request_t request; /* our handoff request */ int last_requested; /* status describes our request rather than our demotion */ long handoff_refuse_log_at; /* when we may log a refused handoff request again */ - ulong peer_floor; /* highest last vote the peer reported, kept across sessions */ - ulong own_floor; /* highest vote this machine signed since it started */ + int engage_pending; /* we paired while running the identity, votor has to engage the failover voting rules, see publish_engage */ + + ulong peer_floor; /* highest last vote the peer reported for the target, kept across sessions */ + ulong own_floor; /* highest vote this machine signed as the target */ /* gossip_ciseen, contact info updates */ ulong gossip_in_idx; /* gossip_ciseen in link */ @@ -243,18 +245,18 @@ struct fd_failover_tile_ctx { /* The admin tile forwards operator commands over the bus and waits for one response frame per request. */ - ulong admin_in_idx; /* admin_failov in link */ - fd_wksp_t * admin_in_mem; /* its workspace */ - ulong admin_in_chunk0; /* its first chunk */ - ulong admin_in_wmark; /* its last chunk */ - ulong admin_out_idx; /* failov_admin out link */ - fd_wksp_t * admin_out_mem; /* its workspace */ - ulong admin_out_chunk0; /* its first chunk */ - ulong admin_out_wmark; /* its last chunk */ - ulong admin_out_chunk; /* next chunk we publish to */ - fd_failover_bus_msg_t bus_req; /* the operator command copied in during_frag */ - int bus_req_fresh; /* bus_req arrived and is not handled yet */ - fd_failover_operator_t bus_operator; /* copied in during_frag */ + ulong admin_in_idx; /* admin_failov in link */ + fd_wksp_t * admin_in_mem; /* its workspace */ + ulong admin_in_chunk0; /* its first chunk */ + ulong admin_in_wmark; /* its last chunk */ + ulong admin_out_idx; /* failov_admin out link */ + fd_wksp_t * admin_out_mem; /* its workspace */ + ulong admin_out_chunk0; /* its first chunk */ + ulong admin_out_wmark; /* its last chunk */ + ulong admin_out_chunk; /* next chunk we publish to */ + fd_failover_bus_msg_t bus_req; /* the operator command copied in during_frag */ + int bus_req_fresh; /* bus_req arrived and is not handled yet */ + fd_failover_operator_t bus_operator; /* copied in during_frag */ fd_failover_id_switch_t id_switch; /* the identity switch we asked the admin tile for */ @@ -273,7 +275,7 @@ struct fd_failover_tile_ctx { ulong adopt_result_id; /* the request it answers */ int adopt_result_fresh; /* adopt_result arrived and is not handled yet */ - /* Unreliable vote history input. Any overrun disables failover. */ + /* Vote history input. */ ulong hist_in_idx; /* votor_hist in link */ fd_wksp_t * hist_in_mem; /* its workspace */ ulong hist_in_chunk0; /* its first chunk */ @@ -281,7 +283,6 @@ struct fd_failover_tile_ctx { int hist_fresh; /* a history frame arrived and is not handled yet */ ulong hist_seen_seq; /* seq of the last votor_hist frag taken in, ULONG_MAX before any */ - int hist_gap; /* a votor_hist frag was skipped since the cached history was built */ /* Votor history and adoption bounds. */ fd_votor_hist_msg_t hist; /* the votor_hist frame copied in during_frag */ @@ -350,21 +351,20 @@ privileged_init( fd_topo_t const * topo, ctx->hist_seen_seq = ULONG_MAX; FD_TEST( fd_rng_secure( &ctx->handoff_base, sizeof(ulong) ) ); - /* The boot identity goes into HELLO and TLS. With failover enabled - it is generated for this boot. Only Sign holds its private key. */ - uchar const * junk_pubkey = fd_keyload_load( tile->failov.identity_key_path, 1 ); + /* The junk key generated at boot goes into HELLO and TLS, and a + handoff switches to it. Only Sign holds its private key. The boot + identity is what we hand off until set-identity or failover promote + picks another. */ + uchar const * junk_pubkey = fd_keyload_load( FD_KEYLOAD_FAILOVER_JUNK, 1 ); fd_memcpy( ctx->hello.junk_pubkey, junk_pubkey, 32UL ); fd_keyload_unload( junk_pubkey, 1 ); - ctx->client_only = !tile->failov.staked_key_path[ 0 ]; - ctx->target_pending = ctx->client_only; - if( !ctx->client_only ) { - uchar const * staked_pubkey = fd_keyload_load( tile->failov.staked_key_path, 1 ); - fd_memcpy( ctx->hello.staked_pubkey, staked_pubkey, 32UL ); - fd_keyload_unload( staked_pubkey, 1 ); - } + uchar const * boot_pubkey = fd_keyload_load( tile->failov.identity_key_path, 1 ); + fd_memcpy( ctx->hello.staked_pubkey, boot_pubkey, 32UL ); + fd_memcpy( ctx->installed, boot_pubkey, 32UL ); + fd_keyload_unload( boot_pubkey, 1 ); if( FD_UNLIKELY( fd_memeq( ctx->hello.junk_pubkey, ctx->hello.staked_pubkey, 32UL ) ) ) { - FD_LOG_ERR(( "the boot identity must differ from the staked [paths.identity_key]" )); + FD_LOG_ERR(( "the failover junk key must differ from [paths.identity_key]" )); } /* The vote key is a base58 pubkey or a keypair file, same as in the @@ -379,10 +379,10 @@ privileged_init( fd_topo_t const * topo, ctx->adopt_anchor = FD_FAILOVER_SLOT_NULL; ctx->empty_vote_after = FD_FAILOVER_SLOT_NULL; - /* Every boot starts passively, as a standby on the junk key. */ - ctx->role = FD_FAILOVER_ROLE_STANDBY; + /* We run the boot identity, which is the target. */ + ctx->holds = 1; ctx->hello.version = (ushort)FD_FAILOVER_VERSION; - ctx->hello.role = (uchar)ctx->role; + ctx->hello.holds = (uchar)ctx->holds; ctx->hello.mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; while( FD_UNLIKELY( !ctx->hello.boot_id ) ) FD_TEST( fd_rng_secure( &ctx->hello.boot_id, sizeof(ulong) ) ); /* The commit field has the hash bytes, a build without one leaves it @@ -402,20 +402,20 @@ privileged_init( fd_topo_t const * topo, ctx->channel = fd_failover_channel_join( fd_failover_channel_new( channel_mem ) ); FD_TEST( ctx->channel ); fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 }; - if( FD_UNLIKELY( !ctx->target_pending && fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, (fd_failover_proof_signer_t){ .ctx=ctx, .sign_fn=sign_proof }, &ctx->hello ) ) ) { - FD_LOG_ERR(( "failover TLS setup failed for the boot identity" )); + if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, (fd_failover_proof_signer_t){ .ctx=ctx, .sign_fn=sign_proof }, &ctx->hello ) ) ) { + FD_LOG_ERR(( "failover TLS setup failed for the junk key" )); } - if( !ctx->client_only ) fd_failover_channel_init_listener( ctx->channel, tile->failov.listen_addr, tile->failov.port ); + fd_failover_channel_init_listener( ctx->channel, tile->failov.listen_addr, tile->failov.port ); ctx->session.state = fd_failover_channel_state( ctx->channel ); char junk_b58 [ FD_BASE58_ENCODED_32_SZ ]; char staked_b58[ FD_BASE58_ENCODED_32_SZ ]; fd_base58_encode_32( ctx->hello.junk_pubkey, NULL, junk_b58 ); fd_base58_encode_32( ctx->hello.staked_pubkey, NULL, staked_b58 ); - if( !ctx->client_only ) FD_LOG_NOTICE(( "failover is on, booting as a standby under junk identity `%s` for staked identity `%s`, listening on `" FD_IP4_ADDR_FMT ":%u`, " - "this machine does not vote until `failover promote` hands it the identity from the active, or `failover promote --force` takes it when no machine holds it", - junk_b58, staked_b58, FD_IP4_ADDR_FMT_ARGS( tile->failov.listen_addr ), (uint)tile->failov.port )); + FD_LOG_NOTICE(( "failover is on, listening on `" FD_IP4_ADDR_FMT ":%u` with junk key `%s`, a handoff from this machine gives up `%s` for the junk key, " + "`failover promote ` here takes the identity in that keypair from the machine running it", + FD_IP4_ADDR_FMT_ARGS( tile->failov.listen_addr ), (uint)tile->failov.port, junk_b58, staked_b58 )); FD_TEST( FD_SCRATCH_ALLOC_FINI( l, scratch_align() )==(ulong)scratch+scratch_footprint( tile ) ); } @@ -521,16 +521,20 @@ sync_session( fd_failover_tile_ctx_t * ctx ) { if( FD_UNLIKELY( state==FD_FAILOVER_SESSION_PAIRED ) ) { fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); ctx->session.peer_boot_id = peer->boot_id; - ctx->session.peer_role = peer->role; + ctx->session.peer_holds = !!peer->holds; if( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_SENT ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_PENDING; if( FD_UNLIKELY( ( ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY || ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ) && - peer->role==(uchar)FD_FAILOVER_ROLE_ACTIVE ) ) ctx->promotion.active_seen = 1; - if( FD_UNLIKELY( peer->role==(uchar)FD_FAILOVER_ROLE_ACTIVE && ( ctx->peer_history.valid ) ) ) - FD_LOG_NOTICE(( "peer boot %016lx authenticated ACTIVE, discarding saved final state from before its voting tenure, keeping coverage floors", peer->boot_id )); - if( FD_UNLIKELY( peer->role==(uchar)FD_FAILOVER_ROLE_ACTIVE ) ) { + peer->holds ) ) ctx->promotion.holder_seen = 1; + if( FD_UNLIKELY( peer->holds && ( ctx->peer_history.valid ) ) ) + FD_LOG_NOTICE(( "peer boot %016lx authenticated running the identity, discarding saved final state from before its voting tenure, keeping coverage floors", peer->boot_id )); + if( FD_UNLIKELY( peer->holds ) ) { ctx->peer_history.valid = 0; } + /* The machine running the identity engages votor's failover voting + rules as soon as it has a peer, so its vote history streams in full + before any handoff request. */ + if( FD_UNLIKELY( ctx->holds ) ) ctx->engage_pending = 1; } } @@ -542,12 +546,12 @@ dial_port( fd_failover_tile_ctx_t const * ctx ) { } static void -set_role( fd_failover_tile_ctx_t * ctx, - ulong role ) { - /* Publish the role reached by the completed identity switch. */ - ctx->role = role; - ctx->hello.role = (uchar)role; - fd_failover_channel_set_role( ctx->channel, role ); +set_holds( fd_failover_tile_ctx_t * ctx, + int holds ) { + /* Publish whether the completed identity switch installed the target. */ + ctx->holds = holds; + ctx->hello.holds = (uchar)holds; + fd_failover_channel_set_holds( ctx->channel, holds ); } /* Sets the replay slot past which the current step is overdue. */ @@ -562,8 +566,8 @@ deadline_start( fd_failover_tile_ctx_t * ctx, /* Sets the slot deadline to slots past the current replay slot, unless it is already set or no replay slot has been seen yet. */ static void -deadline_arm( fd_failover_tile_ctx_t * ctx, - ulong slots ) { +set_step_deadline( fd_failover_tile_ctx_t * ctx, + ulong slots ) { if( FD_LIKELY( ctx->deadline_slot!=FD_FAILOVER_SLOT_NULL ) ) return; if( FD_UNLIKELY( ctx->replay_slot==FD_FAILOVER_SLOT_NULL ) ) return; ctx->deadline_slot = fd_ulong_sat_add( ctx->replay_slot, slots ); @@ -578,9 +582,9 @@ deadline_expired( fd_failover_tile_ctx_t const * ctx ) { /* Sets the promotion deadline in slots and on the monotonic clock. */ static void -promote_deadline_arm( fd_failover_tile_ctx_t * ctx, +set_promote_deadline( fd_failover_tile_ctx_t * ctx, long now ) { - deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + set_step_deadline( ctx, FD_FAILOVER_DEADLINE_SLOTS ); if( FD_UNLIKELY( !ctx->promotion.until ) ) ctx->promotion.until = fd_long_sat_add( now, (long)FD_FAILOVER_DEADLINE_SLOTS*FD_FAILOVER_DEADLINE_SLOT_NANOS ); } @@ -610,7 +614,7 @@ switch_overdue( fd_failover_tile_ctx_t * ctx ) { fd_cstr_printf( demotion, sizeof(demotion), NULL, "handoff %lu", ctx->handoff.id ); FD_LOG_WARNING(( "%s: identity switch %lu to %s is overdue, its outcome is unknown, we keep waiting and refuse another switch, check the admin and sign tile logs", ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? ctx->promotion.label : demotion, - ctx->id_switch.request_id, ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? "staked" : "junk" )); + ctx->id_switch.request_id, ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_STAKED ? "the target identity" : "the junk key" )); } /* Only one control message can be outstanding at a time. Its recipient @@ -680,32 +684,33 @@ finish_reply( fd_failover_tile_ctx_t * ctx, queue_reply( ctx ); } -/* The name of a result the old active sent us for our handoff request, or - the one we sent, and what the operator can do about it. */ +/* The name of a result the machine running the identity sent us for our + handoff request, or the one we sent, and what the operator can do about + it. */ static char const * -relayed_refusal( ulong result, - char const ** hint ) { +control_result( ulong result, + char const ** hint ) { switch( result ) { case FD_ADMINCTL_RESULT_SUCCESS: - *hint = "the old active recorded that we took the identity but this machine is a standby, check `failover status` on both machines"; + *hint = "the other machine recorded that we took the identity but this machine does not run it, check `failover status` on both machines"; return "SUCCESS"; case FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY: - *hint = "the active could not finish this handoff and may have given up the identity, check `failover status` there"; + *hint = "the other machine could not finish this handoff and may have given up the identity, check `failover status` there"; return "PEER_UNREADY"; case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: - *hint = "the active is busy with another transition, run `failover promote` here again once `failover status` there shows it idle"; + *hint = "the other machine is busy with another transition, run `failover promote` here again once `failover status` there shows it idle"; return "IN_PROGRESS"; - case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: - *hint = "the machine we dialed is not the active any more, check `failover status` on both machines"; - return "BAD_ROLE"; + case FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER: + *hint = "the machine we dialed does not run the identity any more, check `failover status` on both machines"; + return "WRONG_HOLDER"; case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: - *hint = "the active has no eligible final vote state to hand over and keeps the identity, its log says why"; + *hint = "the other machine has no eligible final vote state to hand over and keeps the identity, its log says why"; return "NO_FINAL_HISTORY"; case FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND: - *hint = "this machine's replay is too far behind the active's last vote and the active keeps the identity, run `failover promote` here again once this machine has caught up"; + *hint = "this machine's replay is too far behind the other machine's last vote and it keeps the identity, run `failover promote` here again once this machine has caught up"; return "REPLAY_BEHIND"; default: - *hint = "the active answered with a result this machine does not know, run the same Firedancer version on both machines"; + *hint = "the other machine answered with a result this machine does not know, run the same Firedancer version on both machines"; return "UNKNOWN"; } } @@ -719,7 +724,7 @@ control_sent( fd_failover_tile_ctx_t * ctx, if( ctx->handoff.delivery==FD_FAILOVER_DELIVERY_PENDING ) ctx->handoff.delivery = FD_FAILOVER_DELIVERY_SENT; if( now>=ctx->handoff.log_at ) { ctx->handoff.log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); - FD_LOG_NOTICE(( "handoff %lu: sent final state (DEMOTED) to peer boot %016lx, last vote slot %lu, we remain standby waiting for its answer", + FD_LOG_NOTICE(( "handoff %lu: sent final state (DEMOTED) to peer boot %016lx, last vote slot %lu, we run the junk key and wait for its answer", ctx->handoff.id, ctx->handoff.target.boot_id, ctx->last_vote_slot )); } break; @@ -729,7 +734,7 @@ control_sent( fd_failover_tile_ctx_t * ctx, ctx->reply.log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); FD_LOG_NOTICE(( "handoff %lu: sent %s to peer boot %016lx, %s, waiting for its final confirmation", ctx->reply.handoff_id, ctx->tx.type==FD_FAILOVER_MSG_PROMOTE_ACK ? "ACK" : "refusal", - ctx->reply.peer.boot_id, ctx->role==FD_FAILOVER_ROLE_ACTIVE ? "we hold the staked identity" : "we remain standby" )); + ctx->reply.peer.boot_id, ctx->holds ? "we run the identity" : "we do not run the identity" )); } break; case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: { @@ -741,7 +746,7 @@ control_sent( fd_failover_tile_ctx_t * ctx, FD_LOG_NOTICE(( "handoff %lu: sent %s confirmation%s%s%s to peer boot %016lx, closing connection after queued bytes drain", result.handoff_id, result.result==FD_ADMINCTL_RESULT_SUCCESS ? "success" : "refusal", result.result==FD_ADMINCTL_RESULT_SUCCESS ? "" : " (", - result.result==FD_ADMINCTL_RESULT_SUCCESS ? "" : relayed_refusal( result.result, &hint ), + result.result==FD_ADMINCTL_RESULT_SUCCESS ? "" : control_result( result.result, &hint ), result.result==FD_ADMINCTL_RESULT_SUCCESS ? "" : ")", ctx->tx.to.boot_id )); } break; @@ -780,25 +785,32 @@ prepare_history( fd_failover_tile_ctx_t * ctx, ctx->current_history.valid = 1; ctx->current_history.tip = hist->vote_slot; ctx->current_history.sz = state_sz; - ctx->hist_gap = 0; /* a complete snapshot supersedes whatever was skipped */ } -/* consume_slot_done for Alpenglow, it takes in a votor_hist frame. */ +/* Takes in a votor_hist frame: replay and root slots, and the history + of the identity we hand off once the votor streams it in full. */ static void -consume_hist( fd_failover_tile_ctx_t * ctx, - fd_votor_hist_msg_t const * hist ) { +on_votor_hist_msg( fd_failover_tile_ctx_t * ctx, + fd_votor_hist_msg_t const * hist ) { if( FD_LIKELY( hist->replay_slot!=FD_FAILOVER_SLOT_NULL && ( ctx->replay_slot==FD_FAILOVER_SLOT_NULL || hist->replay_slot>ctx->replay_slot ) ) ) ctx->replay_slot = hist->replay_slot; if( FD_LIKELY( hist->root_slot!=FD_FAILOVER_SLOT_NULL ) ) ctx->root_slot = hist->root_slot; - /* Boot-key histories can arrive during preparation or after the switch. */ - if( FD_UNLIKELY( ctx->target_pending || !fd_memeq( hist->identity.uc, ctx->hello.staked_pubkey, 32UL ) ) ) return; - int producing = ctx->role==FD_FAILOVER_ROLE_ACTIVE || ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH; + /* Histories of another key can arrive during preparation or after the + switch. */ + if( FD_UNLIKELY( !fd_memeq( hist->identity.uc, ctx->hello.staked_pubkey, 32UL ) ) ) return; + int producing = ctx->holds || ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH; if( FD_LIKELY( hist->has_vote && hist->vote_slot!=FD_FAILOVER_SLOT_NULL ) ) { /* Every vote here is one we signed, so a later promotion has to cover it. */ ctx->last_vote_slot = hist->vote_slot; if( FD_LIKELY( ctx->own_floor==FD_FAILOVER_SLOT_NULL || hist->vote_slot>ctx->own_floor ) ) ctx->own_floor = hist->vote_slot; prepare_history( ctx, hist ); + } else if( FD_UNLIKELY( producing && hist->vote_slot!=FD_FAILOVER_SLOT_NULL && ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ) ) { + /* The first full history after votor engaged, its tip is a vote we + signed before failover engaged. */ + ctx->last_vote_slot = hist->vote_slot; + if( FD_LIKELY( ctx->own_floor==FD_FAILOVER_SLOT_NULL || hist->vote_slot>ctx->own_floor ) ) ctx->own_floor = hist->vote_slot; + prepare_history( ctx, hist ); } else if( FD_UNLIKELY( producing && ctx->current_history.valid && hist->vote_slot!=FD_FAILOVER_SLOT_NULL && hist->vote_slot>=ctx->last_vote_slot ) ) { /* A frame with no new vote can still change the history, after a LEADER or a vote marked unsent. */ @@ -807,9 +819,10 @@ consume_hist( fd_failover_tile_ctx_t * ctx, } } -/* Sends history or a forced empty-history bound to votor. The response - carries the request ID to distinguish late replies. Returns that ID, - or ULONG_MAX if there was no request to send. */ +/* Sends history or a forced empty-history bound to votor, for the target + identity. The response carries the request ID to distinguish late + replies. Returns that ID, or ULONG_MAX if there was no request to + send. */ static ulong publish_adopt_state( fd_failover_tile_ctx_t * ctx, fd_stem_context_t * stem ) { @@ -817,14 +830,29 @@ publish_adopt_state( fd_failover_tile_ctx_t * ctx, !ctx->promotion.adopt.sz ) ) return ULONG_MAX; if( FD_UNLIKELY( !++ctx->adopt_request_id ) ) ctx->adopt_request_id++; - fd_memcpy( fd_chunk_to_laddr( ctx->adopt_out_mem, ctx->adopt_out_chunk ), ctx->promotion.adopt.state, ctx->promotion.adopt.sz ); + uchar * req = fd_chunk_to_laddr( ctx->adopt_out_mem, ctx->adopt_out_chunk ); + ulong sz = FD_VOTOR_ADOPT_REQ_HDR_SZ+ctx->promotion.adopt.sz; + fd_memcpy( req, ctx->hello.staked_pubkey, FD_VOTOR_ADOPT_REQ_HDR_SZ ); + fd_memcpy( req+FD_VOTOR_ADOPT_REQ_HDR_SZ, ctx->promotion.adopt.state, ctx->promotion.adopt.sz ); ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); - fd_stem_publish( stem, ctx->adopt_out_idx, ctx->adopt_request_id, ctx->adopt_out_chunk, ctx->promotion.adopt.sz, 0UL, tspub, tspub ); - ctx->adopt_out_chunk = fd_dcache_compact_next( ctx->adopt_out_chunk, ctx->promotion.adopt.sz, ctx->adopt_out_chunk0, ctx->adopt_out_wmark ); + fd_stem_publish( stem, ctx->adopt_out_idx, ctx->adopt_request_id, ctx->adopt_out_chunk, sz, 0UL, tspub, tspub ); + ctx->adopt_out_chunk = fd_dcache_compact_next( ctx->adopt_out_chunk, sz, ctx->adopt_out_chunk0, ctx->adopt_out_wmark ); ctx->adopt_result_fresh = 0; return ctx->adopt_request_id; } +/* Tells votor to engage the failover voting rules for the identity we + run. The frame is empty and gets no answer, from then on the + votor_hist stream carries our full history. */ +static void +publish_engage( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem ) { + ctx->engage_pending = 0; + if( FD_UNLIKELY( ctx->adopt_out_idx==ULONG_MAX ) ) return; + ulong tspub = fd_frag_meta_ts_comp( fd_tickcount() ); + fd_stem_publish( stem, ctx->adopt_out_idx, 0UL, ctx->adopt_out_chunk, FD_VOTOR_ADOPT_ENGAGE_SZ, 0UL, tspub, tspub ); +} + /* Asks the admin tile to switch to the given key. Only the public key goes out, the signing tile preloaded both keypairs. Only one request can be in flight, and the reply comes back with the same id so a late @@ -856,8 +884,8 @@ request_switch( fd_failover_tile_ctx_t * ctx, caller should take the result. We drop a reply to another request, and the reply to a switch that set-identity overtook. */ static int -switch_answer( fd_failover_tile_ctx_t * ctx, - ulong nonce ) { +on_switch_resp_msg( fd_failover_tile_ctx_t * ctx, + ulong nonce ) { if( FD_UNLIKELY( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT || nonce!=ctx->id_switch.request_id ) ) { FD_LOG_WARNING(( "dropping a stale identity switch answer" )); @@ -873,15 +901,14 @@ switch_answer( fd_failover_tile_ctx_t * ctx, return 1; } -/* set-identity installed a key while failover is on, so we start over - as if this machine had rebooted with that key. We take a new boot id, - so the other machine treats anything bound to the old one as from a - restarted peer, drop everything in flight, and take the failover - identity and role that set-identity left. A peer with another - failover identity fails HELLO. */ +/* set-identity installed a key, or failover promote picked a + target, so we start over as if this machine had rebooted. We take a + new boot id, so the other machine treats anything bound to the old one + as from a restarted peer, drop everything in flight, and take the + target that admin left. A peer with another target fails HELLO. */ static void -operator_switched( fd_failover_tile_ctx_t * ctx, - fd_failover_operator_t const * operator ) { +on_operator_action_msg( fd_failover_tile_ctx_t * ctx, + fd_failover_operator_t const * operator ) { if( FD_UNLIKELY( operator->epoch<=ctx->id_switch.epoch ) ) return; ctx->id_switch.epoch = operator->epoch; ctx->id_switch.discard = ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT; @@ -892,15 +919,35 @@ operator_switched( fd_failover_tile_ctx_t * ctx, fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); fd_failover_channel_hangup( ctx->channel, now ); + /* Whatever was in flight with the other machine is dropped here, but + the other machine may already have acted on it. */ + int promoting = ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY || + ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT || + ctx->action==FD_FAILOVER_ACTION_PROMOTE_SWITCH; + int requesting = !!ctx->request.id; + int handing = ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE; + int replying = ctx->reply.state==FD_FAILOVER_REPLY_PENDING; + if( FD_UNLIKELY( requesting || handing || promoting || replying ) ) { + char request[ 64 ], handoff[ 64 ], reply[ 64 ]; /* a 20 digit id fits */ + fd_cstr_printf( request, sizeof(request), NULL, " our handoff request %lu,", ctx->request.id ); + fd_cstr_printf( handoff, sizeof(handoff), NULL, " our handoff %lu awaiting its answer,", ctx->handoff.id ); + fd_cstr_printf( reply, sizeof(reply), NULL, " our unsent reply to handoff %lu,", ctx->reply.handoff_id ); + FD_LOG_WARNING(( "the identity changed while failover was busy, dropping%s%s%s%s%s%s the other machine may already run the identity, " + "check `failover status` on both machines", + requesting ? request : "", handing ? handoff : "", + promoting ? " the " : "", promoting ? ctx->promotion.label : "", promoting ? " in progress," : "", + replying ? reply : "" )); + } + fd_memset( &ctx->request, 0, sizeof(ctx->request) ); fd_memset( &ctx->handoff, 0, sizeof(ctx->handoff) ); fd_memset( &ctx->promotion, 0, sizeof(ctx->promotion) ); fd_memset( &ctx->reply, 0, sizeof(ctx->reply) ); fd_memset( &ctx->tx, 0, sizeof(ctx->tx) ); + ctx->engage_pending = 0; ctx->peer_history.valid = 0; ctx->current_history.valid = 0; ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; - ctx->hist_gap = 0; ctx->taken = 0; ctx->last_requested = 0; ctx->cmd_addr = 0U; @@ -914,26 +961,27 @@ operator_switched( fd_failover_tile_ctx_t * ctx, if( FD_UNLIKELY( !fd_memeq( operator->failover_identity, ctx->hello.staked_pubkey, 32UL ) ) ) { ctx->staked_addr = 0U; ctx->staked_seen_at = 0L; + ctx->peer_floor = FD_FAILOVER_SLOT_NULL; + ctx->own_floor = FD_FAILOVER_SLOT_NULL; } ulong boot_id = ctx->hello.boot_id; do FD_TEST( fd_rng_secure( &ctx->hello.boot_id, sizeof(ulong) ) ); while( FD_UNLIKELY( !ctx->hello.boot_id || ctx->hello.boot_id==boot_id ) ); fd_memcpy( ctx->hello.staked_pubkey, operator->failover_identity, 32UL ); - int staked = !fd_memeq( operator->identity, ctx->hello.junk_pubkey, 32UL ); - ctx->role = staked ? FD_FAILOVER_ROLE_ACTIVE : FD_FAILOVER_ROLE_STANDBY; - ctx->hello.role = (uchar)ctx->role; + fd_memcpy( ctx->installed, operator->identity, 32UL ); + ctx->holds = fd_memeq( operator->identity, operator->failover_identity, 32UL ); + ctx->hello.holds = (uchar)ctx->holds; fd_tls_sign_t signer = { .ctx=ctx, .sign_fn=sign_ed25519 }; if( FD_UNLIKELY( fd_failover_channel_set_identity( ctx->channel, ctx->hello.junk_pubkey, signer, (fd_failover_proof_signer_t){ .ctx=ctx, .sign_fn=sign_proof }, &ctx->hello ) ) ) { - FD_LOG_ERR(( "failover TLS setup failed after set-identity" )); + FD_LOG_ERR(( "failover TLS setup failed after the identity changed" )); } - ctx->target_pending = 0; sync_session( ctx ); FD_BASE58_ENCODE_32_BYTES( operator->identity, identity ); FD_BASE58_ENCODE_32_BYTES( operator->failover_identity, failover_identity ); - FD_LOG_WARNING(( "Admin selected failover identity `%s` with installed identity `%s`, this machine is a %s with a new boot %016lx, " - "anything in progress was dropped and the other machine sees a restart", - failover_identity, identity, staked ? "active" : "standby", ctx->hello.boot_id )); + FD_LOG_NOTICE(( "failover now hands off `%s` and this machine runs `%s`, with a new boot %016lx, " + "anything in progress was dropped and the other machine sees a restart", + failover_identity, identity, ctx->hello.boot_id )); } /* Our DEMOTED got its response or cannot get one any more. */ @@ -941,14 +989,14 @@ static void handoff_resolved( fd_failover_tile_ctx_t * ctx, ulong result ) { if( FD_UNLIKELY( ctx->tx.valid && ctx->tx.type==(ushort)FD_FAILOVER_MSG_DEMOTED ) ) ctx->tx.valid = 0; - /* The handoff outcome ends the ACK wait, this machine stays standby. */ + /* The handoff outcome ends the ACK wait, this machine keeps its junk key. */ if( FD_LIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) ctx->action = FD_FAILOVER_ACTION_IDLE; ctx->handoff.delivery = FD_FAILOVER_DELIVERY_NONE; ctx->handoff.result = result; } /* A demotion that ends without sending final state. Nobody can - promote on it, and a requester bound to it is told at once rather than + promote on it, and the machine that asked is told at once rather than at its deadline. */ static void demotion_abort( fd_failover_tile_ctx_t * ctx ) { @@ -968,8 +1016,8 @@ demotion_deadline_start( fd_failover_tile_ctx_t * ctx ) { ctx->handoff.until = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_CHANNEL_IDLE_NANOS ); } -/* Give up the staked identity for the peer that asked, on the session - it asked on. The switch itself is asked for from step_controller, +/* Give up the identity for the peer that asked, on the session it asked + on. The switch itself is asked for from step_controller, DEMOTED goes to the peer boot we see now. */ static void start_demotion( fd_failover_tile_ctx_t * ctx, @@ -987,8 +1035,8 @@ start_demotion( fd_failover_tile_ctx_t * ctx, ctx->action = FD_FAILOVER_ACTION_DEMOTE_SWITCH; } -/* Returns the slot a standby's replay has to reach before it adopts what - we hand over. */ +/* Returns the slot the other machine's replay has to reach before it + adopts what we hand over. */ static ulong ready_slot( fd_failover_tile_ctx_t const * ctx ) { ag_hist_t hist[1]; @@ -996,13 +1044,12 @@ ready_slot( fd_failover_tile_ctx_t const * ctx ) { return hist->anchor; } -/* The final history has to be the history of our last vote, with no history - frag skipped since it was built. */ +/* The final history has to be the history of our last vote. */ static int final_history_ok( fd_failover_tile_ctx_t const * ctx ) { return ctx->current_history.valid && ctx->current_history.sz && ctx->last_vote_slot!=FD_FAILOVER_SLOT_NULL && - ctx->current_history.tip==ctx->last_vote_slot && !ctx->hist_gap; + ctx->current_history.tip==ctx->last_vote_slot; } /* Called once the junk key is installed everywhere. Our latest history is @@ -1010,8 +1057,8 @@ final_history_ok( fd_failover_tile_ctx_t const * ctx ) { static void demotion_switched( fd_failover_tile_ctx_t * ctx ) { /* The junk key is installed and votor has drained. */ - set_role( ctx, FD_FAILOVER_ROLE_STANDBY ); - FD_LOG_NOTICE(( "handoff %lu: the vote history stream drained, we are a standby now", ctx->handoff.id )); + set_holds( ctx, 0 ); + FD_LOG_NOTICE(( "handoff %lu: the vote history stream drained, this machine runs its junk key now", ctx->handoff.id )); int final_ok = final_history_ok( ctx ); @@ -1022,7 +1069,7 @@ demotion_switched( fd_failover_tile_ctx_t * ctx ) { /* We have no final history to hand over, or the one we have is not the history of our last vote. The peer gets no final state and cannot promote. */ - FD_LOG_WARNING(( "handoff %lu has no final vote history for its last vote %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->last_vote_slot )); + FD_LOG_WARNING(( "handoff %lu has no final vote history for its last vote %lu, sending nothing, nobody votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both", ctx->handoff.id, ctx->last_vote_slot )); /* End demotion without sending final state. */ demotion_abort( ctx ); return; @@ -1085,37 +1132,36 @@ control_refusal( ulong result, case FD_FAILOVER_CONTROL_RESULT_INPUT_LOST: *hint = "failover lost an input update and is disabled until restart, validator processing continues"; return "INPUT_LOST"; - case FD_FAILOVER_CONTROL_RESULT_BAD_ROLE: - *hint = "`failover promote` runs on a standby, see `failover status`"; - return "BAD_ROLE"; + case FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER: + *hint = "this machine already runs the identity, `failover promote ` with another keypair is refused on the machine running the identity unless --force"; + return "WRONG_HOLDER"; case FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS: *hint = "a transition or key switch is running, wait for it, `failover status` shows it"; return "IN_PROGRESS"; - case FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS: - *hint = "wait for gossip to show the staked identity, or give the active's address with `failover promote --address`, " - "and if both machines show `role: standby` no machine holds the identity and `failover promote --force` on one of them takes it"; - return "NO_ACTIVE_ADDRESS"; + case FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS: + *hint = "gossip has no address for the machine running the identity yet, gossip refreshes within about a minute, retry or pass `--address`, " + "if `failover status` on every machine shows `runs target identity: no` and no promotion in progress, " + "run `failover promote --force` only on the machine whose status shows a coverage floor, never on both"; + return "NO_HOLDER_ADDRESS"; case FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY: *hint = "the peer could not finish this request, see `failover status` on the peer"; return "PEER_UNREADY"; - case FD_FAILOVER_CONTROL_RESULT_PEER_UNVERIFIED: - *hint = "the peer cannot be verified, --force is required, including first use and restart. Vote history is unknown: it has not been checked. Use `failover promote` here to request a handoff from an active failover peer, or `failover promote --force` only after ensuring every other machine with this identity cannot sign"; - return "PEER_UNVERIFIED"; - case FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE: - *hint = "the peer holds the identity or said so recently, run `failover promote` here, or `failover promote --force` if it cannot sign"; - return "PEER_ACTIVE"; + case FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS: + *hint = "the peer runs the identity or said so recently, run `failover promote` here, or `failover promote --force` if it cannot sign"; + return "PEER_HOLDS"; case FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING: - *hint = "the peer has not answered our handoff, if it shows `role: active` run `failover promote` there to finish it, " - "if it shows `role: standby` nothing votes and `failover promote --force` on one machine takes the identity"; + *hint = "the peer has not answered our handoff, its request pauses at its deadline and `failover promote` on the requester resumes it, " + "`failover promote --force` only when its status shows `runs target identity: no` and no promotion in progress, " + "and only on the machine whose status shows a coverage floor, never both"; return "HANDOFF_PENDING"; case FD_FAILOVER_CONTROL_RESULT_TAKEN: *hint = "the peer took our last handoff and may be voting, run `failover promote` here, or `failover promote --force` if it cannot sign"; return "TAKEN"; case FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN: - *hint = "gossip showed the staked identity at another host within 15 seconds, stop it or wait, or `failover promote --force` if it cannot sign"; + *hint = "gossip showed the identity at another host within 15 seconds, stop it or wait, or `failover promote --force` if it cannot sign"; return "STAKED_SEEN"; case FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY: - *hint = "the active has no eligible final vote state to hand over and keeps the identity, its log explains whether no vote is known or history is missing, check its voting progress before retrying"; + *hint = "the other machine has no eligible final vote state to hand over and keeps the identity, its log explains whether no vote is known or history is missing, check its voting progress before retrying"; return "NO_FINAL_HISTORY"; default: *hint = "the failover tile does not know this command"; @@ -1123,7 +1169,7 @@ control_refusal( ulong result, } } -/* Wait for the old active to confirm our response to its DEMOTED. A +/* Wait for the other machine to confirm our response to its DEMOTED. A request that paused during the promotion dials again, and if the boot we answered restarted, peer_poll ends the request when it pairs with the new one. */ @@ -1134,8 +1180,8 @@ request_wait_result( fd_failover_tile_ctx_t * ctx ) { ctx->request.until = fd_long_sat_add( fd_failover_clock(), FD_FAILOVER_CHANNEL_IDLE_NANOS ); } -/* Stand back down. If the peer's DEMOTED asked for the promotion, we - tell it why and drop the history it handed us, because after a refusal +/* Give up the promotion. If the peer's DEMOTED asked for it, we tell it + why and drop the history it handed us, because after a refusal either machine may recover on its own without reporting its later votes. */ static void @@ -1144,23 +1190,23 @@ reject_promotion( fd_failover_tile_ctx_t * ctx, int bad_history ) { if( FD_UNLIKELY( bad_history && ctx->promotion.source==FD_FAILOVER_SOURCE_PEER ) ) ctx->peer_history.valid = 0; ctx->stuck = 1; - /* Failed local promotion returns to idle on the standby. */ + /* A failed promotion returns to idle without the identity. */ ctx->action = FD_FAILOVER_ACTION_IDLE; if( FD_UNLIKELY( !ctx->promotion.from_peer ) ) { - FD_LOG_WARNING(( "%s: promotion failed (%s), we stay a standby, check that one machine votes and see `failover status`, " + FD_LOG_WARNING(( "%s: promotion failed (%s), this machine does not take the identity, check that one machine votes and see `failover status`, " "run `failover promote --force` again once this machine has caught up and the other cannot sign", ctx->promotion.label, reject_name( reason ) )); return; } ctx->peer_history.valid = 0; ctx->promotion.from_peer = 0; - /* A failed handoff waits for the old active to confirm our refusal. */ + /* A failed handoff waits for the other machine to confirm our refusal. */ request_wait_result( ctx ); if( FD_UNLIKELY( reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { - FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), we stay a standby, another machine may hold the staked identity, " + FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), this machine does not take the identity, another machine may run it, " "check `failover status` on both machines", ctx->promotion.handoff_id, reject_name( reason ) )); } else { - FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), we stay a standby and the peer already gave up the staked identity, " - "nothing votes until `failover promote --force` runs on one machine, check `failover status` on both machines first", + FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), this machine does not take the identity and the peer already gave it up, " + "nothing votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both, check `failover status` on both machines first", ctx->promotion.handoff_id, reject_name( reason ) )); } finish_reply( ctx, &ctx->promotion.peer, ctx->promotion.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason ); @@ -1193,12 +1239,12 @@ replay_ready( fd_failover_tile_ctx_t * ctx ) { ulong floor = ctx->promotion.floor; ctx->empty_vote_after = floor==FD_FAILOVER_SLOT_NULL ? ctx->replay_slot : fd_ulong_max( ctx->replay_slot, floor ); if( FD_UNLIKELY( floor==FD_FAILOVER_SLOT_NULL ) ) { - FD_LOG_WARNING(( "%s: --force accepts empty Alpenglow history, no vote or leader window at or below replay slot %lu, no earlier vote as the staked identity " + FD_LOG_WARNING(( "%s: --force accepts empty Alpenglow history, no vote or leader window at or below replay slot %lu, no earlier vote as the identity " "is known, so a vote the peer cast above that slot is not covered, local vote marks and stronger local bounds remain", ctx->promotion.label, ctx->empty_vote_after )); } else { FD_LOG_WARNING(( "%s: --force accepts empty Alpenglow history, no vote or leader window at or below slot %lu, which covers replay at slot %lu and the " - "last vote known as the staked identity at slot %lu, local vote marks and stronger local bounds remain", + "last vote known as the identity at slot %lu, local vote marks and stronger local bounds remain", ctx->promotion.label, ctx->empty_vote_after, ctx->replay_slot, floor )); } FD_STORE( ulong, ctx->promotion.adopt.state, ctx->empty_vote_after ); @@ -1220,7 +1266,7 @@ log_adopted( fd_failover_tile_ctx_t const * ctx ) { } } -/* Take the staked identity with the history from source. A peer is given +/* Take the identity with the history from source. A peer is given when that member's DEMOTED with handoff_id asked for it, it gets our response. An operator promotion passes NULL and 0. force is promote --force. */ @@ -1250,7 +1296,6 @@ start_promotion( fd_failover_tile_ctx_t * ctx, /* Until this tenure votes, its final history is exactly the adopted one, not a history cached from an earlier tenure. */ ctx->current_history = ctx->promotion.adopt; - ctx->hist_gap = 0; ctx->last_vote_slot = ctx->promotion.adopt.tip; ctx->stuck = 0; promote_start( ctx ); @@ -1303,27 +1348,27 @@ floor_covered( fd_failover_tile_ctx_t const * ctx ) { return floor==FD_FAILOVER_SLOT_NULL || ( tip!=FD_FAILOVER_SLOT_NULL && tip>=floor ); } -/* Whether an active peer authenticated after this promotion started, - which stops it unless it is forced. */ +/* Whether a peer running the identity authenticated after this + promotion started, which stops it unless it is forced. */ static int promote_holder_seen( fd_failover_tile_ctx_t const * ctx ) { if( FD_UNLIKELY( ctx->promotion.force ) ) return 0; - if( FD_UNLIKELY( ctx->promotion.active_seen ) ) return 1; + if( FD_UNLIKELY( ctx->promotion.holder_seen ) ) return 1; return 0; } static void step_demote_drain( fd_failover_tile_ctx_t * ctx ) { - deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + set_step_deadline( ctx, FD_FAILOVER_DEADLINE_SLOTS ); int expired = deadline_expired( ctx ) || ( ctx->handoff.until && fd_failover_clock()>=ctx->handoff.until ); /* The watermark is the votor's output sequence when it stopped signing. */ if( FD_UNLIKELY( !fd_seq_ge( fd_seq_inc( ctx->hist_seen_seq, 1UL ), ctx->id_switch.result.history_watermark ) ) ) { if( FD_UNLIKELY( expired ) ) { - FD_LOG_WARNING(( "handoff %lu: the vote history stream did not report its last votes in time, up to seq %lu, sending nothing, nobody votes until `failover promote --force` runs on one machine", ctx->handoff.id, ctx->id_switch.result.history_watermark )); + FD_LOG_WARNING(( "handoff %lu: the vote history stream did not report its last votes in time, up to seq %lu, sending nothing, nobody votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both", ctx->handoff.id, ctx->id_switch.result.history_watermark )); ctx->id_switch.fresh = 0; ctx->id_switch.overdue = 0; - set_role( ctx, FD_FAILOVER_ROLE_STANDBY ); + set_holds( ctx, 0 ); demotion_abort( ctx ); } return; @@ -1336,13 +1381,13 @@ step_demote_drain( fd_failover_tile_ctx_t * ctx ) { static void step_demote_switch( fd_failover_tile_ctx_t * ctx, fd_stem_context_t * stem ) { - deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + set_step_deadline( ctx, FD_FAILOVER_DEADLINE_SLOTS ); int expired = deadline_expired( ctx ) || ( ctx->handoff.until && fd_failover_clock()>=ctx->handoff.until ); if( FD_UNLIKELY( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.fresh ) ) { if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_JUNK )==ULONG_MAX ) ) { /* The switch request did not go out and we still have the identity, so we send no final state. */ - FD_LOG_WARNING(( "an identity switch could not be requested, remaining the active" )); + FD_LOG_WARNING(( "an identity switch could not be requested, this machine keeps the identity" )); /* End demotion without sending final state. */ demotion_abort( ctx ); return; @@ -1360,13 +1405,15 @@ step_demote_switch( fd_failover_tile_ctx_t * ctx, final state. */ ctx->id_switch.fresh = 0; ctx->id_switch.overdue = 0; - FD_LOG_WARNING(( "the identity switch for handoff %lu failed (%s), we keep the staked identity and stay the active", ctx->handoff.id, - ctx->id_switch.result.result==FD_FAILOVER_SWITCH_ERR_DISABLED ? "failover disabled" : "unknown" )); + FD_LOG_WARNING(( "the identity switch for handoff %lu failed (%s), this machine keeps the identity", ctx->handoff.id, + ctx->id_switch.result.result==FD_FAILOVER_SWITCH_ERR_DISABLED ? "failover disabled" : + ctx->id_switch.result.result==FD_FAILOVER_SWITCH_ERR_AUTHORIZED_VOTER ? "the identity is an authorized voter" : "unknown" )); /* End demotion without sending final state. */ demotion_abort( ctx ); return; } /* The junk key is installed everywhere. */ + fd_memcpy( ctx->installed, ctx->hello.junk_pubkey, 32UL ); FD_LOG_NOTICE(( "handoff %lu: the junk identity is installed, waiting for the vote history stream to drain up to seq %lu", ctx->handoff.id, ctx->id_switch.result.history_watermark )); ctx->action = FD_FAILOVER_ACTION_DEMOTE_DRAIN; demotion_deadline_start( ctx ); @@ -1377,14 +1424,16 @@ static void step_demote_wait_ack( fd_failover_tile_ctx_t * ctx ) { /* If the peer goes quiet we do not undo the demotion, we only raise the alarm. The identity is already gone from this machine. */ - deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + set_step_deadline( ctx, FD_FAILOVER_DEADLINE_SLOTS ); /* A new peer boot cannot respond to this DEMOTED. Its earlier boot may - already have taken the identity and voted before losing its ACK. */ + already have taken the identity and voted before losing its ACK. + The junk key is generated at every boot, so a paired peer with + another boot id is the member's new boot: pairing proved it holds + the same staked key. */ if( FD_UNLIKELY( fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED && - fd_memeq( fd_failover_channel_peer_hello( ctx->channel )->junk_pubkey, ctx->handoff.target.junk, 32UL ) && ctx->session.peer_boot_id!=ctx->handoff.target.boot_id ) ) { - FD_LOG_WARNING(( "the peer restarted before it answered handoff %lu and may have voted before it restarted, neither machine holds the staked identity now, " - "nothing votes until `failover promote --force` runs on one of them", ctx->handoff.id )); + FD_LOG_WARNING(( "the peer restarted before it answered handoff %lu and may have voted before it restarted, neither machine runs the identity now, " + "nothing votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both", ctx->handoff.id )); ctx->peer_history.valid = 0; /* A new peer boot ends this ACK wait without proving whether it voted. */ handoff_resolved( ctx, FD_FAILOVER_HANDOFF_RESTARTED ); @@ -1396,9 +1445,9 @@ step_demote_wait_ack( fd_failover_tile_ctx_t * ctx ) { } if( FD_UNLIKELY( deadline_expired( ctx ) || ( ctx->handoff.until && fd_failover_clock()>=ctx->handoff.until ) ) ) { if( FD_UNLIKELY( !ctx->stuck ) ) - FD_LOG_WARNING(( "the peer has not answered handoff %lu, we remain standby and resend final state when it reconnects, " - "if the peer shows `role: active` run `failover promote` there to finish this handoff, " - "if it shows `role: standby` nothing votes and `failover promote --force` on one machine takes the identity", ctx->handoff.id )); + FD_LOG_WARNING(( "the peer has not answered handoff %lu, we run the junk key, the requester's request pauses at its deadline and " + "`failover promote` on the requester resumes it, `failover promote --force` only when its status shows `runs target identity: no` " + "and no promotion in progress, and only on the machine whose status shows a coverage floor, never both", ctx->handoff.id )); ctx->stuck = 1; } return; @@ -1408,9 +1457,9 @@ static void step_promote_wait_replay( fd_failover_tile_ctx_t * ctx, fd_stem_context_t * stem ) { long now = fd_failover_clock(); - promote_deadline_arm( ctx, now ); + set_promote_deadline( ctx, now ); if( FD_UNLIKELY( promote_expired( ctx, now ) ) ) { - /* Replay timed out, fail the promotion while remaining standby. */ + /* Replay timed out, fail the promotion without taking the identity. */ reject_promotion( ctx, FD_FAILOVER_REJECT_REPLAY_BEHIND, 0 ); return; } @@ -1422,7 +1471,7 @@ step_promote_wait_replay( fd_failover_tile_ctx_t * ctx, } ulong id = publish_adopt_state( ctx, stem ); if( FD_UNLIKELY( id==ULONG_MAX ) ) { - /* End the failed adoption attempt while remaining standby. */ + /* End the failed adoption attempt without taking the identity. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } @@ -1438,9 +1487,9 @@ static void step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, fd_stem_context_t * stem ) { long now = fd_failover_clock(); - promote_deadline_arm( ctx, now ); + set_promote_deadline( ctx, now ); if( FD_UNLIKELY( promote_expired( ctx, now ) ) ) { - /* Adoption timed out, fail promotion while remaining standby. */ + /* Adoption timed out, fail promotion without taking the identity. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } @@ -1449,7 +1498,7 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, if( FD_LIKELY( ctx->replay_slot<=ctx->promotion.retry_slot ) ) return; ulong id = publish_adopt_state( ctx, stem ); if( FD_UNLIKELY( id==ULONG_MAX ) ) { - /* End the failed adoption attempt while remaining standby. */ + /* End the failed adoption attempt without taking the identity. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } @@ -1466,7 +1515,7 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, /* Operator promotion tries a fallback before waiting for more replay. */ if( promote_fallback( ctx, adopt_err_name( ctx->adopt_result.result ) ) ) return; if( FD_UNLIKELY( !ctx->promotion.from_peer ) ) { - /* End the failed adoption attempt while remaining standby. */ + /* End the failed adoption attempt without taking the identity. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } @@ -1493,7 +1542,7 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, /* Operator promotion may retry a different source after a tip mismatch. */ if( promote_fallback( ctx, "adoption did not retain its final vote" ) ) return; FD_LOG_WARNING(( "%s: the adopted vote history ends at slot %lu, the history says %lu, so it is a mismatch", ctx->promotion.label, ctx->adopt_result.vote_slot, ctx->promotion.adopt.tip )); - /* The adopted tip differs, fail promotion without installing the staked key. */ + /* The adopted tip differs, fail promotion without installing the identity. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_MISMATCH, 1 ); return; } @@ -1518,30 +1567,30 @@ step_promote_wait_adopt( fd_failover_tile_ctx_t * ctx, if( FD_UNLIKELY( promote_holder_seen( ctx ) ) ) { /* The identity may be held elsewhere now, so we do not install it too. */ - FD_LOG_WARNING(( "%s: another machine may hold the identity now, a peer authenticated as ACTIVE during this promotion, " - "not installing the staked key, check `failover status` on both machines", ctx->promotion.label )); - /* Another holder was seen, stop before installing the staked key. */ + FD_LOG_WARNING(( "%s: another machine may run the identity now, a peer running it authenticated during this promotion, " + "not installing it, check `failover status` on both machines", ctx->promotion.label )); + /* Another holder was seen, stop before installing the identity. */ reject_promotion( ctx, FD_FAILOVER_REJECT_HOLDS_IDENTITY, 0 ); return; } if( FD_UNLIKELY( ctx->promotion.adopt.tip==FD_FAILOVER_SLOT_NULL ) ) ctx->last_vote_slot = ctx->adopt_result.vote_slot; if( FD_UNLIKELY( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_STAKED )==ULONG_MAX ) ) { - /* The switch was not submitted, fail promotion while remaining standby. */ + /* The switch was not submitted, fail promotion without taking the identity. */ reject_promotion( ctx, FD_FAILOVER_REJECT_ADOPTION_FAILED, 0 ); return; } - FD_LOG_NOTICE(( "%s: asking the admin tile to install the staked identity", ctx->promotion.label )); - /* Adoption passed, wait for admin to install the staked identity. */ + FD_LOG_NOTICE(( "%s: asking the admin tile to install the identity", ctx->promotion.label )); + /* Adoption passed, wait for admin to install the identity. */ ctx->action = FD_FAILOVER_ACTION_PROMOTE_SWITCH; return; } static void step_promote_switch( fd_failover_tile_ctx_t * ctx ) { - deadline_arm( ctx, FD_FAILOVER_DEADLINE_SLOTS ); + set_step_deadline( ctx, FD_FAILOVER_DEADLINE_SLOTS ); if( FD_LIKELY( !ctx->id_switch.fresh ) ) { /* No response yet. Standing down here would tell the peer nobody - promoted while the staked key may well be installed. So we keep + promoted while the identity may well be installed. So we keep waiting and raise stuck past the deadline, the late response still counts. */ if( FD_UNLIKELY( deadline_expired( ctx ) || ( ctx->promotion.until && fd_failover_clock()>=ctx->promotion.until ) ) ) switch_overdue( ctx ); @@ -1550,19 +1599,20 @@ step_promote_switch( fd_failover_tile_ctx_t * ctx ) { ctx->id_switch.fresh = 0; ctx->id_switch.overdue = 0; if( FD_UNLIKELY( ctx->id_switch.result.result!=FD_FAILOVER_SWITCH_OK ) ) { - /* Admin refused the switch, remain standby and fail the promotion. */ + /* Admin refused the switch, fail the promotion without the identity. */ reject_promotion( ctx, FD_FAILOVER_REJECT_SWITCH_FAILED, 0 ); return; } - /* A local recovery changes the role outside a handoff. Reauthenticate - so a background session cannot retain our old standby HELLO. */ + /* A local recovery installs the identity outside a handoff. + Reauthenticate so a background session cannot retain our old HELLO. */ if( !ctx->promotion.from_peer ) { fd_failover_channel_init_dialer( ctx->channel, 0U, 0 ); fd_failover_channel_hangup( ctx->channel, fd_failover_clock() ); sync_session( ctx ); } - /* Admin confirmed the staked identity is installed, become active. */ - set_role( ctx, FD_FAILOVER_ROLE_ACTIVE ); + /* Admin confirmed the identity is installed. */ + fd_memcpy( ctx->installed, ctx->hello.staked_pubkey, 32UL ); + set_holds( ctx, 1 ); /* A new tenure starts, the peer history we kept is older than its votes. */ ctx->peer_history.valid = 0; ctx->taken = 0; @@ -1570,26 +1620,70 @@ step_promote_switch( fd_failover_tile_ctx_t * ctx ) { ctx->action = FD_FAILOVER_ACTION_IDLE; ctx->stuck = 0; if( FD_LIKELY( ctx->promotion.source!=FD_FAILOVER_SOURCE_EMPTY ) ) { - FD_LOG_NOTICE(( "%s: we are the active now, the staked identity is installed and the adopted vote history ends at slot %lu", ctx->promotion.label, ctx->last_vote_slot )); + FD_LOG_NOTICE(( "%s: this machine runs the identity now, the adopted vote history ends at slot %lu", ctx->promotion.label, ctx->last_vote_slot )); } else { - FD_LOG_NOTICE(( "%s: we are the active now, the staked identity is installed with an empty history, no vote at or below slot %lu", ctx->promotion.label, ctx->empty_vote_after )); + FD_LOG_NOTICE(( "%s: this machine runs the identity now with an empty history, no vote at or below slot %lu", ctx->promotion.label, ctx->empty_vote_after )); } if( FD_UNLIKELY( ctx->promotion.from_peer ) ) { - /* The ACK tells the old active its handoff is done. We keep it as + /* The ACK tells the other machine its handoff is done. We keep it as our response, so the same DEMOTED again gets the ACK again. */ ctx->promotion.from_peer = 0; - /* Remain active while the old active confirms our ACK. */ + /* Keep the identity while the other machine confirms our ACK. */ request_wait_result( ctx ); finish_reply( ctx, &ctx->promotion.peer, ctx->promotion.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_ACK, (uchar)FD_FAILOVER_REJECT_NONE ); } return; } +/* Whether we can hand off now to a machine whose replay is at + replay_slot. Returns the refusal code or FD_ADMINCTL_RESULT_SUCCESS. */ +static ulong +handoff_check( fd_failover_tile_ctx_t * ctx, + ulong handoff_id, + ulong replay_slot, + long now ) { + ulong result = FD_ADMINCTL_RESULT_SUCCESS; + if( !final_history_ok( ctx ) ) result = FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY; + /* A machine this far behind could not reach the slot its promotion + waits for before the deadline. */ + else if( replay_slot==FD_FAILOVER_SLOT_NULL || + fd_ulong_sat_add( replay_slot, FD_FAILOVER_DEADLINE_SLOTS )=ctx->handoff_refuse_log_at ) { + ctx->handoff_refuse_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); + char tip[ 32 ], last[ 32 ]; + char const * why = ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ? "no vote is known to the failover controller, it cannot determine whether this machine has not voted or is unable to vote" : + !ctx->current_history.valid || !ctx->current_history.sz ? "a vote is known but its final state is unavailable" : + "the saved final state does not match the latest known vote"; + FD_LOG_WARNING(( "handoff %lu refused: %s (latest known vote %s, saved history tip %s), this machine keeps the identity, check local voting progress and vote-history logs before retrying", + handoff_id, why, slot_text( ctx->last_vote_slot, last ), + slot_text( ctx->current_history.valid ? ctx->current_history.tip : FD_FAILOVER_SLOT_NULL, tip ) )); + } else if( result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND && now>=ctx->handoff_refuse_log_at ) { + ctx->handoff_refuse_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); + char replay[ 32 ]; + FD_LOG_WARNING(( "handoff %lu refused: the other machine's replay is at slot %s, more than %lu slots behind our finality anchor at slot %lu, this machine keeps the identity, " + "run `failover promote` there again once it has caught up", + handoff_id, slot_text( replay_slot, replay ), FD_FAILOVER_DEADLINE_SLOTS, ready_slot( ctx ) )); + } + return result; +} + +/* Starts giving up the identity for the handoff a peer asked for on the + current session. */ +static void +accept_handoff( fd_failover_tile_ctx_t * ctx, + ulong handoff_id ) { + start_demotion( ctx, handoff_id ); + char member[ FD_BASE58_ENCODED_32_SZ ]; + fd_base58_encode_32( ctx->handoff.target.junk, NULL, member ); + FD_LOG_NOTICE(( "handoff %lu: accepted request from member %s boot %016lx, preparing to give up the identity", ctx->handoff.id, member, ctx->handoff.target.boot_id )); +} + /* Advances the current failover step. */ static void step_controller( fd_failover_tile_ctx_t * ctx, fd_stem_context_t * stem ) { if( FD_UNLIKELY( ctx->reply.state==FD_FAILOVER_REPLY_PENDING && !ctx->tx.valid ) ) queue_reply( ctx ); + if( FD_UNLIKELY( ctx->engage_pending ) ) publish_engage( ctx, stem ); if( FD_LIKELY( ctx->action==FD_FAILOVER_ACTION_IDLE ) ) return; switch( ctx->action ) { @@ -1634,15 +1728,15 @@ static char const * request_wait( fd_failover_tile_ctx_t const * ctx ) { switch( ctx->action ) { case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER: - if( !ctx->request.peer.boot_id ) return "no peer authenticated, no handoff request sent, we remain standby"; - if( !ctx->request.send_cnt ) return "peer authenticated, no handoff request sent, we remain standby"; - return "waiting for final state, the peer may still hold the identity"; - case FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY: return "replay catch-up continues locally, we remain standby"; - case FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT: return "adoption continues locally, we remain standby"; - case FD_FAILOVER_ACTION_PROMOTE_SWITCH: return "the staked key switch continues locally, its outcome is still unknown"; + if( !ctx->request.peer.boot_id ) return "no peer authenticated, no handoff request sent, we do not run the identity"; + if( !ctx->request.send_cnt ) return "peer authenticated, no handoff request sent, we do not run the identity"; + return "waiting for final state, the peer may still run the identity"; + case FD_FAILOVER_ACTION_PROMOTE_WAIT_REPLAY: return "replay catch-up continues locally, we do not run the identity"; + case FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT: return "adoption continues locally, we do not run the identity"; + case FD_FAILOVER_ACTION_PROMOTE_SWITCH: return "the identity switch continues locally, its outcome is still unknown"; case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: - return ctx->role==FD_FAILOVER_ROLE_ACTIVE ? "we hold the staked identity, waiting for the peer to confirm our ACK" - : "we remain standby, waiting for the peer to confirm our refusal"; + return ctx->holds ? "we run the identity, waiting for the peer to confirm our ACK" + : "we do not run the identity, waiting for the peer to confirm our refusal"; default: return "the local transition is retained"; } } @@ -1656,15 +1750,15 @@ request_next( fd_failover_tile_ctx_t const * ctx ) { } switch( ctx->action ) { case FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER: - if( !ctx->request.send_cnt ) return "nothing was sent and the active is unchanged, `failover promote` here resumes the request, " - "and if the active is down and cannot sign `failover promote --force` takes over"; - return "the active may already have given up the identity, check `failover status` there, `failover promote` here resumes the request"; + if( !ctx->request.send_cnt ) return "nothing was sent and the other machine still runs the identity, `failover promote` here resumes the request, " + "and if that machine is down and cannot sign `failover promote --force` takes over"; + return "the other machine may already have given up the identity, check `failover status` there, `failover promote` here resumes the request"; case FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT: - if( ctx->role==FD_FAILOVER_ROLE_ACTIVE ) return "this machine holds the staked identity and votes, `failover promote` here asks the old active again to confirm"; - if( ctx->reply.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) return "this machine refused the handoff because another machine may hold the staked identity, " - "check `failover status` on both machines, `failover promote` here asks the old active again to confirm"; - return "neither machine holds the staked identity, `failover promote` here asks the old active again to confirm, " - "and `failover promote --force` on one machine takes the identity"; + if( ctx->holds ) return "this machine runs the identity and votes, `failover promote` here asks the other machine again to confirm"; + if( ctx->reply.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) return "this machine refused the handoff because another machine may run the identity, " + "check `failover status` on both machines, `failover promote` here asks the other machine again to confirm"; + return "neither machine runs the identity, `failover promote` here asks the other machine again to confirm, " + "and `failover promote --force` on the one whose status shows a coverage floor, never both, takes the identity"; default: return "the local promotion goes on and dials again when it ends"; } @@ -1704,195 +1798,208 @@ demoted_answer_bound( fd_failover_tile_ctx_t const * ctx, ctx->action!=FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->action!=FD_FAILOVER_ACTION_DEMOTE_DRAIN; } -/* Handle a controller message. Anything malformed drops the session, - these messages hand off the staked identity so we do not guess at them. */ -static void -handle_control( fd_failover_tile_ctx_t * ctx, - ushort type, - ulong payload_sz, - long now ) { - switch( type ) { +/* The peer's message handlers. Each returns 1 when the message was + handled, including one it ignored on purpose, and 0 when it was + malformed, which drops the session: these messages hand off the + identity so we do not guess at them. */ - case (ushort)FD_FAILOVER_MSG_HANDOFF_REQUEST: { - fd_failover_handoff_request_t request; - if( FD_UNLIKELY( !fd_failover_handoff_request_decode( &request, ctx->rx, payload_sz ) ) ) break; - fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); - if( FD_UNLIKELY( ctx->client_only || request.target_boot_id!=ctx->hello.boot_id ) ) { - handoff_result( ctx, request.handoff_id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); - return; - } - if( FD_UNLIKELY( request.handoff_id==ctx->handoff.id && peer_eq( &ctx->handoff.target, peer->boot_id, peer->junk_pubkey ) ) ) { - /* A duplicate request keeps the same demotion and reuses its eventual response. */ - if( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH || ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN ) return; - if( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) { - if( !ctx->tx.valid ) (void)queue_demoted( ctx ); - } else { - handoff_result( ctx, request.handoff_id, ctx->handoff.result==FD_FAILOVER_HANDOFF_TAKEN - ? FD_ADMINCTL_RESULT_SUCCESS : FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); - } - return; - } - ulong result = FD_ADMINCTL_RESULT_SUCCESS; - if( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT || ctx->tx.valid || ctx->session.close_after_send ) result = FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; - else if( ctx->role!=FD_FAILOVER_ROLE_ACTIVE || peer->role!=FD_FAILOVER_ROLE_STANDBY ) result = FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; - else if( !final_history_ok( ctx ) ) result = FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY; - /* A standby this far behind could not reach the slot its promotion - waits for before the deadline. */ - else if( request.replay_slot==FD_FAILOVER_SLOT_NULL || - fd_ulong_sat_add( request.replay_slot, FD_FAILOVER_DEADLINE_SLOTS )=ctx->handoff_refuse_log_at ) { - ctx->handoff_refuse_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); - char tip[ 32 ], last[ 32 ]; - char const * why = ctx->hist_gap ? "a vote-history update was skipped, the final state cannot be verified" : - ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL ? "no vote is known to the failover controller, it cannot determine whether this machine has not voted or is unable to vote" : - !ctx->current_history.valid || !ctx->current_history.sz ? "a vote is known but its final state is unavailable" : - "the saved final state does not match the latest known vote"; - FD_LOG_WARNING(( "handoff %lu refused: %s (latest known vote %s, saved history tip %s), this machine keeps the staked identity, check local voting progress and vote-history logs before retrying", - request.handoff_id, why, slot_text( ctx->last_vote_slot, last ), - slot_text( ctx->current_history.valid ? ctx->current_history.tip : FD_FAILOVER_SLOT_NULL, tip ) )); - } else if( result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND && now>=ctx->handoff_refuse_log_at ) { - ctx->handoff_refuse_log_at = fd_long_sat_add( now, FD_FAILOVER_LOG_INTERVAL_NANOS ); - char replay[ 32 ]; - FD_LOG_WARNING(( "handoff %lu refused: the standby's replay is at slot %s, more than %lu slots behind %s at slot %lu, this machine keeps the staked identity, " - "run `failover promote` there again once it has caught up", - request.handoff_id, slot_text( request.replay_slot, replay ), FD_FAILOVER_DEADLINE_SLOTS, - "our finality anchor", ready_slot( ctx ) )); - } - handoff_result( ctx, request.handoff_id, result ); - return; +/* HANDOFF_REQUEST, the peer asks us to give up the identity. */ +static int +on_handoff_request_msg( fd_failover_tile_ctx_t * ctx, + ulong payload_sz, + long now ) { + fd_failover_handoff_request_t request; + if( FD_UNLIKELY( !fd_failover_handoff_request_decode( &request, ctx->rx, payload_sz ) ) ) return 0; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + if( FD_UNLIKELY( request.target_boot_id!=ctx->hello.boot_id ) ) { + handoff_result( ctx, request.handoff_id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + return 1; + } + if( FD_UNLIKELY( request.handoff_id==ctx->handoff.id && peer_eq( &ctx->handoff.target, peer->boot_id, peer->junk_pubkey ) ) ) { + /* A duplicate request keeps the same demotion and reuses its eventual response. */ + if( ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH || ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN ) return 1; + if( ctx->handoff.delivery!=FD_FAILOVER_DELIVERY_NONE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) { + if( !ctx->tx.valid ) (void)queue_demoted( ctx ); + } else { + handoff_result( ctx, request.handoff_id, ctx->handoff.result==FD_FAILOVER_HANDOFF_TAKEN + ? FD_ADMINCTL_RESULT_SUCCESS : FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); } - /* An accepted request starts demotion on the active. */ - start_demotion( ctx, request.handoff_id ); - char member[ FD_BASE58_ENCODED_32_SZ ]; - fd_base58_encode_32( peer->junk_pubkey, NULL, member ); - FD_LOG_NOTICE(( "handoff %lu: accepted request from member %s boot %016lx, preparing to give up the staked identity", ctx->handoff.id, member, ctx->handoff.target.boot_id )); - return; - } + return 1; + } + ulong result = FD_ADMINCTL_RESULT_SUCCESS; + if( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT || ctx->tx.valid || ctx->session.close_after_send ) result = FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; + else if( !ctx->holds || peer->holds ) result = FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER; + /* Decline the request without changing the installed identity or action. */ + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + handoff_result( ctx, request.handoff_id, result ); + return 1; + } + result = handoff_check( ctx, request.handoff_id, request.replay_slot, now ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) { + handoff_result( ctx, request.handoff_id, result ); + return 1; + } + accept_handoff( ctx, request.handoff_id ); + return 1; +} - case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: { - fd_failover_handoff_result_t result; - if( FD_UNLIKELY( !fd_failover_handoff_result_decode( &result, ctx->rx, payload_sz ) ) ) break; - fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); - if( FD_UNLIKELY( !ctx->request.id || result.handoff_id!=ctx->request.id || - !peer_eq( &ctx->request.peer, peer->boot_id, peer->junk_pubkey ) ) ) return; - if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && - ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) return; - int failed = result.result!=FD_ADMINCTL_RESULT_SUCCESS || ctx->role!=FD_FAILOVER_ROLE_ACTIVE; - if( FD_UNLIKELY( failed && ctx->role==FD_FAILOVER_ROLE_ACTIVE ) ) { - FD_LOG_WARNING(( "handoff %lu: the old active did not record that we took the identity and may hold it again, check `failover status` on both machines at once", result.handoff_id )); - } else if( FD_UNLIKELY( failed && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && ctx->reply.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { - FD_LOG_WARNING(( "handoff %lu ended, this machine refused the final state (%s) because another machine may hold the staked identity, and the old active " - "recorded it, check `failover status` on both machines", result.handoff_id, reject_name( ctx->reply.reason ) )); - } else if( FD_UNLIKELY( failed && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) { - FD_LOG_WARNING(( "handoff %lu ended, this machine refused the final state (%s) and the old active recorded it, neither machine holds the staked identity, " - "nothing votes until `failover promote --force` runs on one of them", result.handoff_id, reject_name( ctx->reply.reason ) )); - } else if( FD_UNLIKELY( failed ) ) { - char const * hint; - char const * name = relayed_refusal( result.result, &hint ); - FD_LOG_WARNING(( "handoff %lu ended (%s), %s", result.handoff_id, name, hint )); - } - /* A bound final result ends the request without changing our installed identity. */ - request_end( ctx, now, failed ? FD_FAILOVER_HANDOFF_DECLINED : FD_FAILOVER_HANDOFF_TAKEN ); - if( !failed ) - FD_LOG_NOTICE(( "handoff %lu complete: the old active confirmed our ACK, we hold the staked identity, connection closed", result.handoff_id )); - else - FD_LOG_NOTICE(( "handoff %lu: request ended, %s, connection closed", result.handoff_id, - ctx->role==FD_FAILOVER_ROLE_ACTIVE ? "we still hold the staked identity" : "we remain standby" )); - return; - } +/* HANDOFF_RESULT, the peer recorded how our request ended. */ +static int +on_handoff_result_msg( fd_failover_tile_ctx_t * ctx, + ulong payload_sz, + long now ) { + fd_failover_handoff_result_t result; + if( FD_UNLIKELY( !fd_failover_handoff_result_decode( &result, ctx->rx, payload_sz ) ) ) return 0; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + if( FD_UNLIKELY( !ctx->request.id || result.handoff_id!=ctx->request.id || + !peer_eq( &ctx->request.peer, peer->boot_id, peer->junk_pubkey ) ) ) return 1; + if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && + ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) return 1; + int failed = result.result!=FD_ADMINCTL_RESULT_SUCCESS || !ctx->holds; + if( FD_UNLIKELY( failed && ctx->holds ) ) { + FD_LOG_WARNING(( "handoff %lu: the other machine did not record that we took the identity and may run it again, check `failover status` on both machines at once", result.handoff_id )); + } else if( FD_UNLIKELY( failed && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && ctx->reply.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { + FD_LOG_WARNING(( "handoff %lu ended, this machine refused the final state (%s) because another machine may run the identity, and the other machine " + "recorded it, check `failover status` on both machines", result.handoff_id, reject_name( ctx->reply.reason ) )); + } else if( FD_UNLIKELY( failed && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) { + FD_LOG_WARNING(( "handoff %lu ended, this machine refused the final state (%s) and the other machine recorded it, neither machine runs the identity, " + "nothing votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both", result.handoff_id, reject_name( ctx->reply.reason ) )); + } else if( FD_UNLIKELY( failed && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY && + ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && ctx->request.send_cntrequest.send_cnt )); + return 1; + } else if( FD_UNLIKELY( failed ) ) { + char const * hint; + char const * name = control_result( result.result, &hint ); + FD_LOG_WARNING(( "handoff %lu ended (%s), %s", result.handoff_id, name, hint )); + } + /* A bound final result ends the request without changing our installed identity. */ + request_end( ctx, now, failed ? FD_FAILOVER_HANDOFF_DECLINED : FD_FAILOVER_HANDOFF_TAKEN ); + if( !failed ) + FD_LOG_NOTICE(( "handoff %lu complete: the other machine confirmed our ACK, we run the identity, connection closed", result.handoff_id )); + else + FD_LOG_NOTICE(( "handoff %lu: request ended, %s, connection closed", result.handoff_id, + ctx->holds ? "we still run the identity" : "we do not run the identity" )); + return 1; +} - case (ushort)FD_FAILOVER_MSG_DEMOTED: { - /* The peer says it can no longer sign and hands us its history. */ - fd_failover_demoted_t demoted; - if( FD_UNLIKELY( !fd_failover_demoted_decode( &demoted, ctx->rx, payload_sz ) ) ) break; - /* HELLO pairs only members in the same mode, so a DEMOTED in the - other mode is malformed. */ - if( FD_UNLIKELY( demoted.mode!=(uchar)FD_FAILOVER_MODE_ALPENGLOW ) ) break; - ulong boot_id = ctx->session.peer_boot_id; - fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); - /* Bind before deduplication. An unsolicited DEMOTED cannot change - this operation or replace its cached response. */ - if( FD_UNLIKELY( !ctx->request.id || ctx->request.id!=demoted.handoff_id || - !peer_eq( &ctx->request.peer, boot_id, peer->junk_pubkey ) ) ) return; - if( FD_UNLIKELY( demoted.target_boot_id!=ctx->hello.boot_id ) ) break; - /* The same DEMOTED again. Once we are done with it we repeat our - response. */ - if( FD_UNLIKELY( ctx->promotion.from_peer && ctx->promotion.peer.boot_id==boot_id && ctx->promotion.handoff_id==demoted.handoff_id ) ) return; - if( FD_UNLIKELY( ctx->reply.state!=FD_FAILOVER_REPLY_NONE && ctx->reply.handoff_id==demoted.handoff_id && peer_eq( &ctx->reply.peer, boot_id, peer->junk_pubkey ) ) ) { - queue_reply( ctx ); - return; - } - fd_failover_history_t history = { .valid=1, .tip=demoted.last_vote_slot, .sz=(ulong)demoted.state_len }; - fd_memcpy( history.state, ctx->rx+sizeof(fd_failover_demoted_t), history.sz ); - uchar reason = FD_FAILOVER_REJECT_NONE; - char const * why = NULL; - if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_STANDBY ) ) { - reason = FD_FAILOVER_REJECT_HOLDS_IDENTITY; - why = "we hold the staked identity"; - } else if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || - ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) { - reason = FD_FAILOVER_REJECT_BUSY; - why = "a transition or key switch is running here"; - } - if( FD_UNLIKELY( reason!=FD_FAILOVER_REJECT_NONE ) ) { - FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), %s, check `failover status` on both machines", demoted.handoff_id, reject_name( reason ), why )); - finish_reply( ctx, &ctx->request.peer, demoted.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason ); - return; - } - FD_LOG_NOTICE(( "handoff %lu: accepted the peer's final vote history ending at slot %lu, waiting for local adoption before taking the identity", demoted.handoff_id, history.tip )); - if( ctx->peer_floor==FD_FAILOVER_SLOT_NULL || history.tip>ctx->peer_floor ) ctx->peer_floor = history.tip; - ctx->peer_history = history; - ctx->session.peer_role = FD_FAILOVER_ROLE_STANDBY; - /* The bound final state moves the standby from peer wait to replay wait. */ - start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &ctx->request.peer, demoted.handoff_id, 0 ); - return; - } +/* DEMOTED, the peer gave up the identity and hands us its final history. */ +static int +on_demoted_msg( fd_failover_tile_ctx_t * ctx, + ulong payload_sz ) { + /* The peer says it can no longer sign and hands us its history. */ + fd_failover_demoted_t demoted; + if( FD_UNLIKELY( !fd_failover_demoted_decode( &demoted, ctx->rx, payload_sz ) ) ) return 0; + /* HELLO pairs only members in the same mode, so a DEMOTED in the + other mode is malformed. */ + if( FD_UNLIKELY( demoted.mode!=(uchar)FD_FAILOVER_MODE_ALPENGLOW ) ) return 0; + ulong boot_id = ctx->session.peer_boot_id; + fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + /* Bind before deduplication. An unsolicited DEMOTED cannot change + this operation or replace its cached response. */ + if( FD_UNLIKELY( !ctx->request.id || ctx->request.id!=demoted.handoff_id || + !peer_eq( &ctx->request.peer, boot_id, peer->junk_pubkey ) ) ) return 1; + if( FD_UNLIKELY( demoted.target_boot_id!=ctx->hello.boot_id ) ) return 0; + /* The same DEMOTED again. Once we are done with it we repeat our + response. */ + if( FD_UNLIKELY( ctx->promotion.from_peer && ctx->promotion.peer.boot_id==boot_id && ctx->promotion.handoff_id==demoted.handoff_id ) ) return 1; + if( FD_UNLIKELY( ctx->reply.state!=FD_FAILOVER_REPLY_NONE && ctx->reply.handoff_id==demoted.handoff_id && peer_eq( &ctx->reply.peer, boot_id, peer->junk_pubkey ) ) ) { + queue_reply( ctx ); + return 1; + } + fd_failover_history_t history = { .valid=1, .tip=demoted.last_vote_slot, .sz=(ulong)demoted.state_len }; + fd_memcpy( history.state, ctx->rx+sizeof(fd_failover_demoted_t), history.sz ); + uchar reason = FD_FAILOVER_REJECT_NONE; + char const * why = NULL; + if( FD_UNLIKELY( ctx->holds ) ) { + reason = FD_FAILOVER_REJECT_HOLDS_IDENTITY; + why = "we run the identity"; + } else if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || + ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) { + reason = FD_FAILOVER_REJECT_BUSY; + why = "a transition or key switch is running here"; + } + if( FD_UNLIKELY( reason!=FD_FAILOVER_REJECT_NONE ) ) { + FD_LOG_WARNING(( "refusing the peer's handoff %lu (%s), %s, check `failover status` on both machines", demoted.handoff_id, reject_name( reason ), why )); + finish_reply( ctx, &ctx->request.peer, demoted.handoff_id, (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED, reason ); + return 1; + } + FD_LOG_NOTICE(( "handoff %lu: accepted the peer's final vote history ending at slot %lu, waiting for local adoption before taking the identity", demoted.handoff_id, history.tip )); + if( ctx->peer_floor==FD_FAILOVER_SLOT_NULL || history.tip>ctx->peer_floor ) ctx->peer_floor = history.tip; + ctx->peer_history = history; + ctx->session.peer_holds = 0; + /* The bound final state moves us from peer wait to replay wait. */ + start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &ctx->request.peer, demoted.handoff_id, 0 ); + return 1; +} - case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK: { - /* The peer took the identity, so our DEMOTED did its job. An ACK - after the deadline still counts, and we ignore an ACK we are not - waiting for rather than drop the session. */ - fd_failover_promote_ack_t ack; - if( FD_UNLIKELY( !fd_failover_promote_ack_decode( &ack, ctx->rx, payload_sz ) ) ) break; - if( FD_UNLIKELY( !demoted_answer_bound( ctx, ack.handoff_id ) ) ) return; - FD_LOG_NOTICE(( "handoff %lu: received ACK, the peer took the staked identity, we stay a standby", ack.handoff_id )); - /* The peer ACK ends the demotion wait and records that it took the identity. */ - handoff_resolved( ctx, FD_FAILOVER_HANDOFF_TAKEN ); - ctx->taken = 1; - ctx->session.peer_role = FD_FAILOVER_ROLE_ACTIVE; - ctx->stuck = 0; - /* The peer votes past the final history we handed it from here on. */ - ctx->peer_history.valid = 0; - handoff_result( ctx, ack.handoff_id, FD_ADMINCTL_RESULT_SUCCESS ); - return; - } +/* PROMOTE_ACK, the peer took the identity we handed off. */ +static int +on_promote_ack_msg( fd_failover_tile_ctx_t * ctx, + ulong payload_sz ) { + /* The peer took the identity, so our DEMOTED did its job. An ACK + after the deadline still counts, and we ignore an ACK we are not + waiting for rather than drop the session. */ + fd_failover_promote_ack_t ack; + if( FD_UNLIKELY( !fd_failover_promote_ack_decode( &ack, ctx->rx, payload_sz ) ) ) return 0; + if( FD_UNLIKELY( !demoted_answer_bound( ctx, ack.handoff_id ) ) ) return 1; + FD_LOG_NOTICE(( "handoff %lu: received ACK, the peer took the identity, we keep the junk key", ack.handoff_id )); + /* The peer ACK ends the demotion wait and records that it took the identity. */ + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_TAKEN ); + ctx->taken = 1; + ctx->session.peer_holds = 1; + ctx->stuck = 0; + /* The peer votes past the final history we handed it from here on. */ + ctx->peer_history.valid = 0; + handoff_result( ctx, ack.handoff_id, FD_ADMINCTL_RESULT_SUCCESS ); + return 1; +} - case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: { - /* The peer did not take the identity. Either machine may now recover - without telling the other, so we drop any history the peer handed us - earlier. */ - fd_failover_promote_rejected_t rej; - if( FD_UNLIKELY( !fd_failover_promote_rejected_decode( &rej, ctx->rx, payload_sz ) ) ) break; - if( FD_UNLIKELY( !demoted_answer_bound( ctx, rej.handoff_id ) ) ) return; - if( FD_UNLIKELY( rej.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { - FD_LOG_WARNING(( "the peer declined handoff %lu (%s), it saw another machine that may hold the staked identity, this machine gave it up and " - "stays a standby, check `failover status` on both machines and the peer's log", rej.handoff_id, reject_name( rej.reason ) )); - } else { - FD_LOG_WARNING(( "the peer declined handoff %lu (%s), neither machine holds the staked identity now, nothing votes until `failover promote --force` runs on one of them, " - "see the peer's log first", rej.handoff_id, reject_name( rej.reason ) )); - } - ctx->peer_history.valid = 0; - /* The peer refusal ends the demotion wait, this machine stays standby. */ - handoff_resolved( ctx, FD_FAILOVER_HANDOFF_DECLINED ); - ctx->stuck = 1; - handoff_result( ctx, rej.handoff_id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); - return; +/* PROMOTE_REJECTED, the peer did not take the identity we handed off. */ +static int +on_promote_rejected_msg( fd_failover_tile_ctx_t * ctx, + ulong payload_sz ) { + /* The peer did not take the identity. Either machine may now recover + without telling the other, so we drop any history the peer handed us + earlier. */ + fd_failover_promote_rejected_t rej; + if( FD_UNLIKELY( !fd_failover_promote_rejected_decode( &rej, ctx->rx, payload_sz ) ) ) return 0; + if( FD_UNLIKELY( !demoted_answer_bound( ctx, rej.handoff_id ) ) ) return 1; + if( FD_UNLIKELY( rej.reason==FD_FAILOVER_REJECT_HOLDS_IDENTITY ) ) { + FD_LOG_WARNING(( "the peer declined handoff %lu (%s), it saw another machine that may run the identity, this machine gave it up, " + "check `failover status` on both machines and the peer's log", rej.handoff_id, reject_name( rej.reason ) )); + } else { + FD_LOG_WARNING(( "the peer declined handoff %lu (%s), neither machine runs the identity now, nothing votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both, " + "see the peer's log first", rej.handoff_id, reject_name( rej.reason ) )); } + ctx->peer_history.valid = 0; + /* The peer refusal ends the demotion wait, this machine keeps its junk key. */ + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_DECLINED ); + ctx->stuck = 1; + handoff_result( ctx, rej.handoff_id, FD_FAILOVER_CONTROL_RESULT_PEER_UNREADY ); + return 1; +} - default: break; +/* Handle a controller message. Anything malformed drops the session, + these messages hand off the identity so we do not guess at them. */ +static void +handle_control( fd_failover_tile_ctx_t * ctx, + ushort type, + ulong payload_sz, + long now ) { + int handled; + switch( type ) { + case (ushort)FD_FAILOVER_MSG_HANDOFF_REQUEST: handled = on_handoff_request_msg ( ctx, payload_sz, now ); break; + case (ushort)FD_FAILOVER_MSG_HANDOFF_RESULT: handled = on_handoff_result_msg ( ctx, payload_sz, now ); break; + case (ushort)FD_FAILOVER_MSG_DEMOTED: handled = on_demoted_msg ( ctx, payload_sz ); break; + case (ushort)FD_FAILOVER_MSG_PROMOTE_ACK: handled = on_promote_ack_msg ( ctx, payload_sz ); break; + case (ushort)FD_FAILOVER_MSG_PROMOTE_REJECTED: handled = on_promote_rejected_msg( ctx, payload_sz ); break; + default: handled = 0; break; } + if( FD_LIKELY( handled ) ) return; /* Malformed control traffic drops the connection and retains local work. */ fd_failover_channel_protocol_error( ctx->channel, now ); @@ -1922,20 +2029,22 @@ request_end( fd_failover_tile_ctx_t * ctx, sync_session( ctx ); } -/* An idle standby authenticates ahead of an operator command. Merely - pairing never creates a request or changes the installed identity. */ +/* An idle machine that does not run the identity authenticates to the + one that does ahead of an operator command. Merely pairing never + creates a request or changes the installed identity. */ static void preconnect( fd_failover_tile_ctx_t * ctx, long now ) { if( ctx->request.id || ctx->stuck || ctx->tx.valid || ctx->session.close_after_send || ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) return; - /* A stale gossip address may name a validator that now boots passive. - Do not keep that standby session open. Retain the dial backoff. */ - if( ctx->role==FD_FAILOVER_ROLE_STANDBY && + /* A stale gossip address may name a validator that does not run the + identity any more. Do not keep that session open. Retain the dial + backoff. */ + if( !ctx->holds && fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED && - fd_failover_channel_peer_hello( ctx->channel )->role!=FD_FAILOVER_ROLE_ACTIVE ) + !fd_failover_channel_peer_hello( ctx->channel )->holds ) fd_failover_channel_hangup( ctx->channel, now ); - int ready = !ctx->target_pending && ctx->role==FD_FAILOVER_ROLE_STANDBY && + int ready = !ctx->holds && ctx->staked_addr && ctx->staked_seen_at && now>=ctx->staked_seen_at && now-ctx->staked_seen_atchannel, 0U, 0 ); @@ -1946,10 +2055,15 @@ preconnect( fd_failover_tile_ctx_t * ctx, /* An operator command is required before sending a handoff request. */ static void peer_poll( fd_failover_tile_ctx_t * ctx, + fd_stem_context_t * stem, long now, int * charge_busy ) { preconnect( ctx, now ); sync_session( ctx ); + /* A pairing seen here engages the votor before any frame on the new + session is read, else a request carried by that session can arrive + before the votor streams its first full history. */ + if( FD_UNLIKELY( ctx->engage_pending ) ) publish_engage( ctx, stem ); if( FD_UNLIKELY( ctx->request.id && ctx->request.until && now>=ctx->request.until && ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) ) ) { @@ -1972,44 +2086,48 @@ peer_poll( fd_failover_tile_ctx_t * ctx, ulong payload_sz; if( FD_UNLIKELY( fd_failover_channel_poll( ctx->channel, now, charge_busy, &type, ctx->rx, &payload_sz ) ) ) { sync_session( ctx ); + /* The frame may have come on the session that just paired. */ + if( FD_UNLIKELY( ctx->engage_pending ) ) publish_engage( ctx, stem ); handle_control( ctx, type, payload_sz, now ); } sync_session( ctx ); - if( FD_LIKELY( !ctx->request.id || !ctx->request.until || ctx->request.sent || - fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ) ) return; + if( FD_LIKELY( !ctx->request.id || fd_failover_channel_state( ctx->channel )!=FD_FAILOVER_SESSION_PAIRED ) ) return; fd_failover_hello_t const * peer = fd_failover_channel_peer_hello( ctx->channel ); + /* The peer restarted before it confirmed our answer, so it can no + longer confirm this handoff and the wait ends. A paused request + ends too, else the old holder's next request is refused as in + progress until the operator runs `failover promote` here. */ + if( FD_UNLIKELY( ctx->request.peer.boot_id && !peer_eq( &ctx->request.peer, peer->boot_id, peer->junk_pubkey ) && + ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && + ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ) ) { + int holds = ctx->holds; + int peer_holds = !!peer->holds; + FD_LOG_WARNING(( "handoff %lu: the peer restarted before it confirmed our %s, ending the request, %s, %s", ctx->request.id, + holds ? "ACK" : "refusal", holds ? "we run the identity" : "we do not run the identity", + holds && peer_holds ? "and its new boot says it runs the identity too, check `failover status` on both machines at once" : + holds ? "its new boot does not run the identity" : + peer_holds ? "its new boot runs the identity" : + "its new boot does not run it either, nothing votes until `failover promote --force` runs on the one whose status shows a coverage floor, never both" )); + request_end( ctx, now, holds ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ); + return; + } + if( FD_LIKELY( !ctx->request.until || ctx->request.sent ) ) return; if( FD_UNLIKELY( !ctx->request.peer.boot_id ) ) { - if( FD_UNLIKELY( peer->role!=FD_FAILOVER_ROLE_ACTIVE ) ) { - FD_LOG_WARNING(( "handoff %lu dialed a standby at `" FD_IP4_ADDR_FMT ":%hu`, give the active's address with `failover promote --address`, " - "and if both machines show `role: standby` no machine holds the identity and `failover promote --force` on one of them takes it", + if( FD_UNLIKELY( !peer->holds ) ) { + FD_LOG_WARNING(( "handoff %lu dialed `" FD_IP4_ADDR_FMT ":%hu`, which does not run the identity, give the address of the machine running it with `failover promote --address`, " + "and if both machines show `runs target identity: no` no machine runs it and `failover promote --force` on the one whose status shows a coverage floor, never both, takes it", ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), dial_port( ctx ) )); /* End the peer wait without changing the installed identity. */ - request_end( ctx, now, FD_FAILOVER_HANDOFF_NOT_ACTIVE ); + request_end( ctx, now, FD_FAILOVER_HANDOFF_NOT_HOLDER ); return; } - /* Bind the request to the first authenticated active member and boot. */ + /* Bind the request to the first authenticated member and boot running + the identity. */ ctx->request.peer.boot_id = peer->boot_id; fd_memcpy( ctx->request.peer.junk, peer->junk_pubkey, 32UL ); } else if( FD_UNLIKELY( !peer_eq( &ctx->request.peer, peer->boot_id, peer->junk_pubkey ) ) ) { - /* The peer restarted before it confirmed our answer, so it can no - longer confirm this handoff and the wait ends. */ - if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && - ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && - fd_memeq( peer->junk_pubkey, ctx->request.peer.junk, 32UL ) ) ) { - int active = ctx->role==FD_FAILOVER_ROLE_ACTIVE; - int peer_active = peer->role==FD_FAILOVER_ROLE_ACTIVE; - FD_LOG_WARNING(( "handoff %lu: the peer restarted before it confirmed our %s, ending the request, %s, %s", ctx->request.id, - active ? "ACK" : "refusal", active ? "we hold the staked identity" : "we remain standby", - active && peer_active ? "and its new boot says it is active too, check `failover status` on both machines at once" : - active ? "its new boot is a standby" : - peer_active ? "its new boot is the active" : - "its new boot is a standby too, nothing votes until `failover promote --force` runs on one machine" )); - request_end( ctx, now, active ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ); - return; - } - /* A fresh boot has a new junk key. Keep the old binding until the - operator ends this request, without accepting another peer's final - state or acknowledgement for it. */ + /* Keep the old binding until the operator ends this request, without + accepting another boot's final state or acknowledgement for it. */ ctx->request.peer_restarted = 1; request_pause( ctx, now, "authenticated member or boot changed" ); return; @@ -2041,7 +2159,7 @@ promote_guard( fd_failover_tile_ctx_t const * ctx, if( FD_UNLIKELY( ctx->input_lost ) ) return FD_FAILOVER_CONTROL_RESULT_INPUT_LOST; int paired = fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED; - if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_STANDBY ) ) return FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; + if( FD_UNLIKELY( ctx->holds ) ) return FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER; if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK && !force ) ) return FD_FAILOVER_CONTROL_RESULT_HANDOFF_PENDING; int request_wait = ctx->request.id && ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || @@ -2052,13 +2170,13 @@ promote_guard( fd_failover_tile_ctx_t const * ctx, if( FD_UNLIKELY( force ) ) return FD_ADMINCTL_RESULT_SUCCESS; /* The peer took the identity in our last handoff. */ if( FD_UNLIKELY( ctx->taken ) ) return FD_FAILOVER_CONTROL_RESULT_TAKEN; - if( FD_UNLIKELY( paired && fd_failover_channel_peer_hello( ctx->channel )->role==FD_FAILOVER_ROLE_ACTIVE ) ) - return FD_FAILOVER_CONTROL_RESULT_PEER_ACTIVE; - /* Another host is publishing the staked identity in gossip. */ + if( FD_UNLIKELY( paired && fd_failover_channel_peer_hello( ctx->channel )->holds ) ) + return FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS; + /* Another host is publishing the identity in gossip. */ if( FD_UNLIKELY( ctx->staked_seen_at && now>=ctx->staked_seen_at && now-ctx->staked_seen_ataction==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); } -/* Run an operator command. If we refuse it we say why. */ +/* `failover promote` without --force: ask the machine running the + identity to hand it over, or resume a paused request. */ static ulong -apply_control( fd_failover_tile_ctx_t * ctx, - fd_adminctl_failover_req_t const * req, - long now ) { - if( FD_UNLIKELY( ctx->input_lost ) ) return FD_FAILOVER_CONTROL_RESULT_INPUT_LOST; - if( FD_UNLIKELY( ctx->target_pending && req->cmd!=FD_ADMINCTL_FAILOVER_CMD_STATUS ) ) return FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED; - switch( req->cmd ) { - - case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: { - int paused = request_paused( ctx ) && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT; - if( FD_UNLIKELY( paused && ctx->request.peer_restarted ) ) { - if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) { - /* Local adoption is already settled. This command ends only - our wait, without confirming anything on the old peer. */ - int active = ctx->role==FD_FAILOVER_ROLE_ACTIVE; - FD_LOG_WARNING(( "handoff %lu: the authenticated peer changed before it confirmed our answer, ending the confirmation wait, %s, the old peer's acknowledgement remains unknown", - ctx->request.id, active ? "we hold the staked identity" : "we remain standby" )); - request_end( ctx, now, active ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ); - return FD_ADMINCTL_RESULT_SUCCESS; - } - FD_LOG_NOTICE(( "handoff %lu was bound to active boot %016lx, the authenticated peer changed, ending it and starting a new handoff", ctx->request.id, ctx->request.peer.boot_id )); - request_end( ctx, now, FD_FAILOVER_HANDOFF_RESTARTED ); - paused = 0; - } - if( FD_UNLIKELY( paused ) ) { - /* Refresh the destination while retaining the authenticated member - and boot. An explicit address takes precedence over gossip. */ - ctx->request.until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); - ctx->request.sent = 0; - ctx->stuck = 0; - if( req->addr ) ctx->cmd_addr = req->addr; - if( req->port ) ctx->cmd_port = req->port; - if( ctx->cmd_addr ) ctx->request.addr = ctx->cmd_addr; - else if( ctx->staked_addr ) ctx->request.addr = ctx->staked_addr; - fd_failover_channel_init_dialer( ctx->channel, ctx->request.addr, dial_port( ctx ) ); - FD_LOG_NOTICE(( "resuming handoff %lu at `" FD_IP4_ADDR_FMT ":%hu` from %s, %s, %s", - ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), dial_port( ctx ), - ctx->cmd_addr ? "`failover promote --address`" : "gossip", - ctx->request.peer.boot_id ? "same authenticated member and boot" : "active has not authenticated yet", - request_wait( ctx ) )); +control_cmd_handoff( fd_failover_tile_ctx_t * ctx, + fd_adminctl_failover_req_t const * req, + long now ) { + int paused = request_paused( ctx ) && ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT; + if( FD_UNLIKELY( paused && ctx->request.peer_restarted ) ) { + if( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) { + /* Local adoption is already settled. This command ends only + our wait, without confirming anything on the old peer. */ + FD_LOG_WARNING(( "handoff %lu: the authenticated peer changed before it confirmed our answer, ending the confirmation wait, %s, the old peer's acknowledgement remains unknown", + ctx->request.id, ctx->holds ? "we run the identity" : "we do not run the identity" )); + request_end( ctx, now, ctx->holds ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ); return FD_ADMINCTL_RESULT_SUCCESS; } - if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) - return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; - if( FD_UNLIKELY( ctx->role!=FD_FAILOVER_ROLE_STANDBY ) ) return FD_FAILOVER_CONTROL_RESULT_BAD_ROLE; - uint addr = req->addr ? req->addr : ctx->staked_addr; - if( FD_UNLIKELY( !addr ) ) return FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS; - ctx->cmd_addr = req->addr; - ctx->cmd_port = req->port; - fd_memset( &ctx->request, 0, sizeof(ctx->request) ); - ctx->request.id = ctx->handoff_base+(++ctx->handoff_cnt); - if( FD_UNLIKELY( !ctx->request.id ) ) ctx->request.id = ++ctx->handoff_cnt; - ctx->request.last_id = ctx->request.id; - ctx->request.result = FD_FAILOVER_HANDOFF_PENDING; - ctx->request.addr = addr; - ctx->request.until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); - ctx->last_requested = 1; - ctx->stuck = 0; - /* An idle standby starts a new handoff and waits for the active. */ - ctx->action = FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER; - ctx->session.close_after_send = 0; - fd_failover_channel_init_dialer( ctx->channel, addr, dial_port( ctx ) ); - FD_LOG_NOTICE(( "requesting handoff %lu from the active at `" FD_IP4_ADDR_FMT ":%hu` using %s, authenticating the peer before requesting final state", - ctx->request.id, FD_IP4_ADDR_FMT_ARGS( addr ), dial_port( ctx ), ctx->cmd_addr ? "`failover promote --address`" : "gossip" )); + FD_LOG_NOTICE(( "handoff %lu was bound to peer boot %016lx, the authenticated peer changed, ending it and starting a new handoff", ctx->request.id, ctx->request.peer.boot_id )); + request_end( ctx, now, FD_FAILOVER_HANDOFF_RESTARTED ); + paused = 0; + } + if( FD_UNLIKELY( paused ) ) { + /* Refresh the destination while retaining the authenticated member + and boot. An explicit address takes precedence over gossip. */ + ctx->request.until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); + ctx->request.sent = 0; + ctx->stuck = 0; + if( req->addr ) ctx->cmd_addr = req->addr; + if( req->port ) ctx->cmd_port = req->port; + if( ctx->cmd_addr ) ctx->request.addr = ctx->cmd_addr; + else if( ctx->staked_addr ) ctx->request.addr = ctx->staked_addr; + fd_failover_channel_init_dialer( ctx->channel, ctx->request.addr, dial_port( ctx ) ); + FD_LOG_NOTICE(( "resuming handoff %lu at `" FD_IP4_ADDR_FMT ":%hu` from %s, %s, %s", + ctx->request.id, FD_IP4_ADDR_FMT_ARGS( ctx->request.addr ), dial_port( ctx ), + ctx->cmd_addr ? "`failover promote --address`" : "gossip", + ctx->request.peer.boot_id ? "same authenticated member and boot" : "the other machine has not authenticated yet", + request_wait( ctx ) )); return FD_ADMINCTL_RESULT_SUCCESS; } + if( FD_UNLIKELY( ctx->action!=FD_FAILOVER_ACTION_IDLE || ctx->id_switch.pending_key!=FD_FAILOVER_SWITCH_KEY_CNT ) ) + return FD_FAILOVER_CONTROL_RESULT_IN_PROGRESS; + if( FD_UNLIKELY( ctx->holds ) ) return FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER; + uint addr = req->addr ? req->addr : ctx->staked_addr; + if( FD_UNLIKELY( !addr ) ) return FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS; + ctx->cmd_addr = req->addr; + ctx->cmd_port = req->port; + fd_memset( &ctx->request, 0, sizeof(ctx->request) ); + ctx->request.id = ctx->handoff_base+(++ctx->handoff_cnt); + if( FD_UNLIKELY( !ctx->request.id ) ) ctx->request.id = ++ctx->handoff_cnt; + ctx->request.last_id = ctx->request.id; + ctx->request.result = FD_FAILOVER_HANDOFF_PENDING; + ctx->request.addr = addr; + ctx->request.until = fd_long_sat_add( now, FD_FAILOVER_CHANNEL_IDLE_NANOS ); + ctx->last_requested = 1; + ctx->stuck = 0; + /* Start a new handoff and wait for the machine running the identity. */ + ctx->action = FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER; + ctx->session.close_after_send = 0; + fd_failover_channel_init_dialer( ctx->channel, addr, dial_port( ctx ) ); + FD_LOG_NOTICE(( "requesting handoff %lu from the machine running the identity at `" FD_IP4_ADDR_FMT ":%hu` using %s, authenticating the peer before requesting final state", + ctx->request.id, FD_IP4_ADDR_FMT_ARGS( addr ), dial_port( ctx ), ctx->cmd_addr ? "`failover promote --address`" : "gossip" )); + return FD_ADMINCTL_RESULT_SUCCESS; +} - case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: { - /* --force takes over without dialing the peer and accepts incomplete - or empty history. */ - int force = !!( req->flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ); - ulong result = promote_guard( ctx, now, force ); - if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) return result; - ulong source = promote_source( ctx ); - /* The operator fenced the peer. Only a request with no local - adoption or key switch in flight can be cancelled for recovery. */ - if( force && ctx->role==FD_FAILOVER_ROLE_STANDBY && ctx->request.id && - ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) && - ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ) { - FD_LOG_NOTICE(( "handoff %lu cancelled by `failover promote --force` on the operator's word that the peer cannot sign, proceeding with local recovery", ctx->request.id )); - /* Forced recovery ends the peer wait without changing the installed identity. */ - request_end( ctx, now, FD_FAILOVER_HANDOFF_CANCELLED ); - } - if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) { - FD_LOG_WARNING(( "`failover promote --force` stops waiting for the peer's answer to handoff %lu, discarding saved final state because the peer may have voted, coverage floors retained", ctx->handoff.id )); - ctx->peer_history.valid = 0; - /* Forced recovery ends the ACK wait before starting local promotion. */ - handoff_resolved( ctx, FD_FAILOVER_HANDOFF_CANCELLED ); - } - if( force ) FD_LOG_WARNING(( "operator promotion: --force asserts the peer cannot sign, peer checks are bypassed and incomplete or empty vote history is permitted if needed" )); - if( source==FD_FAILOVER_SOURCE_EMPTY ) { - fd_failover_history_t const * saved = &ctx->peer_history; - char const * stored = "the stored peer vote history"; - if( saved->valid ) { - ulong floor = coverage_floor( ctx ); - if( floor!=FD_FAILOVER_SLOT_NULL && saved->tiptip, floor )); - else if( ctx->root_slot!=FD_FAILOVER_SLOT_NULL && saved->tip<=ctx->root_slot ) - FD_LOG_NOTICE(( "operator promotion: %s ends at slot %lu at or below the local root %lu, skipping it", stored, saved->tip, ctx->root_slot )); - } - FD_LOG_NOTICE(( "operator promotion: no eligible saved final state is available, selecting an empty history" )); - } else { - fd_failover_history_t const * saved = &ctx->peer_history; - FD_LOG_NOTICE(( "operator promotion: selecting %s through slot %lu, the %s must validate and adopt it before the staked identity is installed", - source_name( source ), saved->tip, "votor" )); +/* `failover promote --force`: take the identity without the peer. */ +static ulong +control_cmd_promote( fd_failover_tile_ctx_t * ctx, + fd_adminctl_failover_req_t const * req, + long now ) { + /* --force takes over without dialing the peer and accepts incomplete + or empty history. */ + int force = !!( req->flags & FD_ADMINCTL_FAILOVER_FLAG_FORCE ); + /* Admin sends an unforced promote as a handoff, so this one did not + come through it. */ + if( FD_UNLIKELY( !force ) ) { + FD_LOG_WARNING(( "refusing a promote without --force, it did not come through admin" )); + return FD_ADMINCTL_RESULT_UNKNOWN_COMMAND; + } + ulong result = promote_guard( ctx, now, force ); + if( FD_UNLIKELY( result!=FD_ADMINCTL_RESULT_SUCCESS ) ) return result; + ulong source = promote_source( ctx ); + /* The operator fenced the peer. Only a request with no local + adoption or key switch in flight can be cancelled for recovery. */ + if( force && !ctx->holds && ctx->request.id && + ( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER || ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ) && + ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ) { + FD_LOG_NOTICE(( "handoff %lu cancelled by `failover promote --force` on the operator's word that the peer cannot sign, proceeding with local recovery", ctx->request.id )); + /* Forced recovery ends the peer wait without changing the installed identity. */ + request_end( ctx, now, FD_FAILOVER_HANDOFF_CANCELLED ); + } + if( FD_UNLIKELY( ctx->action==FD_FAILOVER_ACTION_DEMOTE_WAIT_ACK ) ) { + FD_LOG_WARNING(( "`failover promote --force` stops waiting for the peer's answer to handoff %lu, discarding saved final state because the peer may have voted, coverage floors retained", ctx->handoff.id )); + ctx->peer_history.valid = 0; + /* Forced recovery ends the ACK wait before starting local promotion. */ + handoff_resolved( ctx, FD_FAILOVER_HANDOFF_CANCELLED ); + } + if( force ) FD_LOG_WARNING(( "operator promotion: --force asserts the peer cannot sign, peer checks are bypassed and incomplete or empty vote history is permitted if needed" )); + if( source==FD_FAILOVER_SOURCE_EMPTY ) { + fd_failover_history_t const * saved = &ctx->peer_history; + char const * stored = "the stored peer vote history"; + if( saved->valid ) { + ulong floor = coverage_floor( ctx ); + if( floor!=FD_FAILOVER_SLOT_NULL && saved->tiptip, floor )); + else if( ctx->root_slot!=FD_FAILOVER_SLOT_NULL && saved->tip<=ctx->root_slot ) + FD_LOG_NOTICE(( "operator promotion: %s ends at slot %lu at or below the local root %lu, skipping it", stored, saved->tip, ctx->root_slot )); } - /* The accepted operator command starts the replay and adoption sequence. */ - start_promotion( ctx, source, NULL, 0UL, force ); - return FD_ADMINCTL_RESULT_SUCCESS; + FD_LOG_NOTICE(( "operator promotion: no eligible saved final state is available, selecting an empty history" )); + } else { + fd_failover_history_t const * saved = &ctx->peer_history; + FD_LOG_NOTICE(( "operator promotion: selecting %s through slot %lu, the %s must validate and adopt it before the identity is installed", + source_name( source ), saved->tip, "votor" )); } + /* The accepted operator command starts the replay and adoption sequence. */ + start_promotion( ctx, source, NULL, 0UL, force ); + return FD_ADMINCTL_RESULT_SUCCESS; +} - default: break; +/* Run an operator command. If we refuse it we say why. */ +static ulong +apply_control( fd_failover_tile_ctx_t * ctx, + fd_adminctl_failover_req_t const * req, + long now ) { + if( FD_UNLIKELY( ctx->input_lost ) ) return FD_FAILOVER_CONTROL_RESULT_INPUT_LOST; + switch( req->cmd ) { + case FD_ADMINCTL_FAILOVER_CMD_HANDOFF: return control_cmd_handoff( ctx, req, now ); + case FD_ADMINCTL_FAILOVER_CMD_PROMOTE: return control_cmd_promote( ctx, req, now ); + default: return FD_ADMINCTL_RESULT_UNSUPPORTED; } - return FD_ADMINCTL_RESULT_UNSUPPORTED; } /* The `failover status` response. */ @@ -2203,14 +2334,14 @@ status_snapshot( fd_failover_tile_ctx_t const * ctx, long now, fd_adminctl_failover_status_resp_t * resp ) { fd_adminctl_failover_status_resp_init( resp ); - resp->enabled = ctx->client_only ? 2 : 1; - resp->role = (uchar)ctx->role; + resp->enabled = 1; + resp->holds = (uchar)ctx->holds; resp->action = (uchar)ctx->action; resp->stuck = (uchar)!!ctx->stuck; resp->link_state = (uchar)fd_failover_channel_state( ctx->channel ); if( FD_UNLIKELY( fd_failover_channel_state( ctx->channel )==FD_FAILOVER_SESSION_PAIRED ) ) { - resp->peer_role_valid = 1; - resp->peer_role = (uchar)ctx->session.peer_role; + resp->peer_holds_valid = 1; + resp->peer_holds = (uchar)ctx->session.peer_holds; } resp->peer_boot_id = ctx->session.peer_boot_id; resp->peer_addr = ctx->request.id ? ctx->request.addr : ctx->staked_addr; @@ -2220,10 +2351,12 @@ status_snapshot( fd_failover_tile_ctx_t const * ctx, resp->handoff_result = (uchar)( ctx->last_requested ? ctx->request.result : ctx->handoff.result ); resp->promote_source = (uchar)promote_source( ctx ); resp->promote_floor = coverage_floor( ctx ); - resp->promote_result = ctx->input_lost ? FD_FAILOVER_CONTROL_RESULT_INPUT_LOST : - ctx->target_pending ? FD_FAILOVER_CONTROL_RESULT_KEY_REQUIRED : promote_guard( ctx, now, 0 ); + resp->promote_result = ctx->input_lost ? FD_FAILOVER_CONTROL_RESULT_INPUT_LOST : promote_guard( ctx, now, 0 ); resp->mode = ctx->hello.mode; resp->request_paused = (uchar)request_paused( ctx ); + fd_memcpy( resp->target, ctx->hello.staked_pubkey, 32UL ); + fd_memcpy( resp->installed, ctx->installed, 32UL ); + fd_memcpy( resp->junk, ctx->hello.junk_pubkey, 32UL ); } /* Respond to one bus request. The admin tile validated the ABI already. @@ -2256,11 +2389,13 @@ serve_bus_request( fd_failover_tile_ctx_t * ctx, } ulong handoff_id = 0UL; if( FD_LIKELY( result==FD_ADMINCTL_RESULT_SUCCESS ) ) { - if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) handoff_id = ctx->request.id; + /* Ending a paused confirmation wait zeroes request.id and keeps + last_id, which is the handoff the command acted on. */ + if( req.cmd==FD_ADMINCTL_FAILOVER_CMD_HANDOFF ) handoff_id = ctx->request.id ? ctx->request.id : ctx->request.last_id; } fd_adminctl_failover_control_resp_t response = { .version = FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, - .role = (uchar)ctx->role, + .holds = (uchar)ctx->holds, .action = (uchar)ctx->action, .handoff_id = handoff_id, }; @@ -2273,11 +2408,11 @@ serve_bus_request( fd_failover_tile_ctx_t * ctx, ctx->admin_out_chunk = fd_dcache_compact_next( ctx->admin_out_chunk, sizeof(*out), ctx->admin_out_chunk0, ctx->admin_out_wmark ); } -/* The active's address comes from gossip, from the newest staked - contact info that is not ours, which is how the active shows up. */ +/* The address of the machine running the identity comes from gossip, + from the newest contact info for the identity that is not ours. */ static void -gossip_commit( fd_failover_tile_ctx_t * ctx, - long now ) { +on_contact_info_msg( fd_failover_tile_ctx_t * ctx, + long now ) { uint addr = ctx->gossip_socket.addr; if( FD_UNLIKELY( !addr ) ) return; if( FD_LIKELY( !fd_memeq( ctx->gossip_origin, ctx->hello.staked_pubkey, 32UL ) ) ) return; @@ -2286,16 +2421,16 @@ gossip_commit( fd_failover_tile_ctx_t * ctx, /* Gossip may be the only sign of a peer tenure, so we drop the peer's history for good, not only while the contact info is fresh. */ if( FD_UNLIKELY( ctx->peer_history.valid ) ) - FD_LOG_NOTICE(( "gossip shows the staked identity at another host, discarding saved final state from before its voting tenure, keeping coverage floors" )); + FD_LOG_NOTICE(( "gossip shows the identity at another host, discarding saved final state from before its voting tenure, keeping coverage floors" )); ctx->peer_history.valid = 0; if( FD_LIKELY( addr==ctx->staked_addr ) ) return; uint old_addr = ctx->staked_addr; ctx->staked_addr = addr; if( FD_LIKELY( !old_addr ) ) { - FD_LOG_NOTICE(( "gossip shows the staked identity at `" FD_IP4_ADDR_FMT "`, a handoff from this machine dials it", FD_IP4_ADDR_FMT_ARGS( addr ) )); + FD_LOG_NOTICE(( "gossip shows the identity at `" FD_IP4_ADDR_FMT "`, a handoff from this machine dials it", FD_IP4_ADDR_FMT_ARGS( addr ) )); } else { - FD_LOG_NOTICE(( "gossip shows a new address for the staked identity, `" FD_IP4_ADDR_FMT "` before and `" FD_IP4_ADDR_FMT "` now, the next handoff uses the new address", + FD_LOG_NOTICE(( "gossip shows a new address for the identity, `" FD_IP4_ADDR_FMT "` before and `" FD_IP4_ADDR_FMT "` now, the next handoff uses the new address", FD_IP4_ADDR_FMT_ARGS( old_addr ), FD_IP4_ADDR_FMT_ARGS( addr ) )); } } @@ -2317,7 +2452,7 @@ input_overrun( fd_failover_tile_ctx_t * ctx ) { static inline int before_frag( fd_failover_tile_ctx_t * ctx, ulong in_idx, - ulong seq, + ulong seq FD_PARAM_UNUSED, ulong sig ) { if( FD_UNLIKELY( ctx->input_lost ) ) return in_idx!=ctx->admin_in_idx || sig!=FD_FAILOVER_BUS_REQUEST; if( FD_LIKELY( in_idx==ctx->gossip_in_idx ) ) { @@ -2325,14 +2460,13 @@ before_frag( fd_failover_tile_ctx_t * ctx, } if( FD_LIKELY( in_idx==ctx->hist_in_idx ) ) { /* Every history frame counts toward the drain watermark after the - stem validates it. A skipped frame may have contained a vote. */ - if( FD_UNLIKELY( ctx->hist_seen_seq!=ULONG_MAX && seq!=fd_seq_inc( ctx->hist_seen_seq, 1UL ) ) ) ctx->hist_gap = 1; + stem validates it. */ return 0; } if( FD_UNLIKELY( in_idx==ctx->admin_in_idx ) ) { /* Commands, switch responses and set-identity. Dropping a switch response here would leave the switch hanging forever. */ - return sig!=FD_FAILOVER_BUS_REQUEST && sig!=FD_FAILOVER_BUS_SWITCH_RESP && sig!=FD_FAILOVER_BUS_OPERATOR; + return sig!=FD_FAILOVER_BUS_REQUEST && sig!=FD_FAILOVER_BUS_SWITCH_RESP && sig!=FD_FAILOVER_BUS_OPERATOR_ACTION; } return 0; } @@ -2363,7 +2497,7 @@ during_frag( fd_failover_tile_ctx_t * ctx, ctx->id_switch.response_nonce = msg->nonce; return; } - if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_OPERATOR ) ) { + if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_OPERATOR_ACTION ) ) { fd_memcpy( &ctx->bus_operator, msg->payload, sizeof(ctx->bus_operator) ); return; } @@ -2421,12 +2555,12 @@ after_frag( fd_failover_tile_ctx_t * ctx, if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_SWITCH_RESP ) ) { /* The drain still reads the last switch result, so we copy a response into it only after the stem and nonce checks pass. */ - if( FD_LIKELY( switch_answer( ctx, ctx->id_switch.response_nonce ) ) ) ctx->id_switch.result = ctx->id_switch.response; - if( FD_UNLIKELY( ctx->id_switch.response.result==FD_FAILOVER_SWITCH_ERR_STALE ) ) operator_switched( ctx, &ctx->id_switch.response.operator ); + if( FD_LIKELY( on_switch_resp_msg( ctx, ctx->id_switch.response_nonce ) ) ) ctx->id_switch.result = ctx->id_switch.response; + if( FD_UNLIKELY( ctx->id_switch.response.result==FD_FAILOVER_SWITCH_ERR_STALE ) ) on_operator_action_msg( ctx, &ctx->id_switch.response.operator ); return; } - if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_OPERATOR ) ) { - operator_switched( ctx, &ctx->bus_operator ); + if( FD_UNLIKELY( sig==FD_FAILOVER_BUS_OPERATOR_ACTION ) ) { + on_operator_action_msg( ctx, &ctx->bus_operator ); return; } /* Send the response from after_credit, where a publish credit is available. */ @@ -2439,7 +2573,7 @@ after_frag( fd_failover_tile_ctx_t * ctx, return; } if( FD_UNLIKELY( in_idx!=ctx->gossip_in_idx ) ) return; - gossip_commit( ctx, fd_failover_clock() ); + on_contact_info_msg( ctx, fd_failover_clock() ); } static inline void @@ -2459,14 +2593,14 @@ after_credit( fd_failover_tile_ctx_t * ctx, } if( FD_UNLIKELY( ctx->hist_fresh ) ) { ctx->hist_fresh = 0; - consume_hist( ctx, &ctx->hist ); + on_votor_hist_msg( ctx, &ctx->hist ); } step_controller( ctx, stem ); long now = fd_failover_clock(); - if( FD_LIKELY( !ctx->target_pending && now>=ctx->poll_at ) ) { + if( FD_LIKELY( now>=ctx->poll_at ) ) { int busy = 0; - peer_poll( ctx, now, &busy ); + peer_poll( ctx, stem, now, &busy ); ctx->poll_at = busy ? now : now+FD_FAILOVER_TILE_POLL_NANOS; if( busy ) *charge_busy = 1; } @@ -2479,7 +2613,7 @@ after_credit( fd_failover_tile_ctx_t * ctx, if( FD_UNLIKELY( ctx->stuck!=ctx->stuck_logged ) ) { ctx->stuck_logged = ctx->stuck; - if( !ctx->stuck ) FD_LOG_NOTICE(( "failover is no longer stuck, %s", ctx->role==FD_FAILOVER_ROLE_ACTIVE ? "this machine is active" : "this machine is standby" )); + if( !ctx->stuck ) FD_LOG_NOTICE(( "failover is no longer stuck, %s", ctx->holds ? "this machine runs the identity" : "this machine does not run the identity" )); } } @@ -2537,6 +2671,9 @@ populate_allowed_fds( fd_topo_t const * topo, #include "../../disco/stem/fd_stem.c" +/* A serialized history fits the DEMOTED payload. */ +FD_STATIC_ASSERT( AG_HIST_SER_MAX<=FD_FAILOVER_ALPENGLOW_STATE_MAX, alpenglow_state_max ); + fd_topo_run_tile_t fd_tile_failov = { .name = "failov", .keep_host_networking = 1, diff --git a/src/discof/failover/fd_failover_wire.c b/src/discof/failover/fd_failover_wire.c index bc6d2d8d9a9..7e8915734a1 100644 --- a/src/discof/failover/fd_failover_wire.c +++ b/src/discof/failover/fd_failover_wire.c @@ -2,7 +2,7 @@ #include -static ulong const payload_max[ FD_FAILOVER_MSG_RESERVED ] = { +static ulong const payload_max[ FD_FAILOVER_MSG_CNT ] = { sizeof(fd_failover_hello_t), sizeof(fd_failover_handoff_request_t), FD_FAILOVER_DEMOTED_PAYLOAD_MAX, @@ -12,21 +12,13 @@ static ulong const payload_max[ FD_FAILOVER_MSG_RESERVED ] = { 0UL /* KEEPALIVE */ }; -void -fd_failover_wire_session_init( fd_failover_wire_session_t * session ) { - session->tx_seq = 0UL; - session->rx_seq = 0UL; -} - ulong -fd_failover_wire_encode( fd_failover_wire_session_t * session, - uchar * out, - ushort type, - uchar const * payload, - ulong payload_sz ) { - if( FD_UNLIKELY( type>=FD_FAILOVER_MSG_RESERVED || - payload_sz>payload_max[type] || - session->tx_seq==ULONG_MAX ) ) return 0UL; +fd_failover_wire_encode( uchar * out, + ushort type, + uchar const * payload, + ulong payload_sz ) { + if( FD_UNLIKELY( type>=FD_FAILOVER_MSG_CNT || + payload_sz>payload_max[type] ) ) return 0UL; ulong frame_sz = FD_FAILOVER_FRAME_HDR_SZ+payload_sz; @@ -34,23 +26,20 @@ fd_failover_wire_encode( fd_failover_wire_session_t * session, .version = (ushort)FD_FAILOVER_VERSION, .len = (uint)frame_sz, .type = type, - .seq = session->tx_seq, }; fd_memcpy( out, &hdr, FD_FAILOVER_FRAME_HDR_SZ ); if( FD_LIKELY( payload_sz ) ) fd_memcpy( out+FD_FAILOVER_FRAME_HDR_SZ, payload, payload_sz ); - session->tx_seq++; return frame_sz; } int -fd_failover_wire_decode( fd_failover_wire_session_t * session, - uchar const * buf, - ulong buf_sz, - ushort * out_type, - uchar const ** out_payload, - ulong * out_payload_sz, - ulong * out_frame_sz ) { +fd_failover_wire_decode( uchar const * buf, + ulong buf_sz, + ushort * out_type, + uchar const ** out_payload, + ulong * out_payload_sz, + ulong * out_frame_sz ) { if( FD_UNLIKELY( buf_szrx_seq==ULONG_MAX || hdr.seq!=session->rx_seq ) ) return FD_FAILOVER_WIRE_ERR_SEQ; - if( FD_UNLIKELY( hdr.type>=FD_FAILOVER_MSG_RESERVED ) ) return FD_FAILOVER_WIRE_ERR_TYPE; + if( FD_UNLIKELY( hdr.type>=FD_FAILOVER_MSG_CNT ) ) return FD_FAILOVER_WIRE_ERR_TYPE; if( FD_UNLIKELY( payload_sz>payload_max[hdr.type] ) ) return FD_FAILOVER_WIRE_ERR_SZ; if( FD_UNLIKELY( buf_szrx_seq++; *out_type = hdr.type; *out_payload = buf+FD_FAILOVER_FRAME_HDR_SZ; *out_payload_sz = payload_sz; diff --git a/src/discof/failover/fd_failover_wire.h b/src/discof/failover/fd_failover_wire.h index 9218df6fbc4..9583f6d7fd1 100644 --- a/src/discof/failover/fd_failover_wire.h +++ b/src/discof/failover/fd_failover_wire.h @@ -8,7 +8,7 @@ #include "fd_failover_proto.h" /* Frame layout bounds */ -#define FD_FAILOVER_FRAME_HDR_SZ (16UL) +#define FD_FAILOVER_FRAME_HDR_SZ (8UL) #define FD_FAILOVER_PAYLOAD_MAX (65536UL) #define FD_FAILOVER_FRAME_MAX (FD_FAILOVER_FRAME_HDR_SZ+FD_FAILOVER_PAYLOAD_MAX) @@ -18,52 +18,36 @@ #define FD_FAILOVER_WIRE_ERR_VERSION (-2) #define FD_FAILOVER_WIRE_ERR_SZ (-3) #define FD_FAILOVER_WIRE_ERR_TYPE (-4) -#define FD_FAILOVER_WIRE_ERR_SEQ (-5) struct __attribute__((packed)) fd_failover_frame_hdr { ushort version; /* FD_FAILOVER_VERSION */ uint len; /* total frame length in bytes */ ushort type; /* FD_FAILOVER_MSG_* */ - ulong seq; /* per direction, starts at zero */ }; typedef struct fd_failover_frame_hdr fd_failover_frame_hdr_t; FD_STATIC_ASSERT( sizeof(fd_failover_frame_hdr_t)==FD_FAILOVER_FRAME_HDR_SZ, wire_layout ); -/* Sequence counters belong to one TLS connection. */ -struct fd_failover_wire_session { - ulong tx_seq; - ulong rx_seq; -}; - -typedef struct fd_failover_wire_session fd_failover_wire_session_t; - FD_PROTOTYPES_BEGIN -/* Resets the sequence counters for a new TLS connection. */ -void -fd_failover_wire_session_init( fd_failover_wire_session_t * session ); - /* Writes one frame and payload into out, which must hold at least FD_FAILOVER_FRAME_MAX bytes. */ ulong -fd_failover_wire_encode( fd_failover_wire_session_t * session, - uchar * out, - ushort type, - uchar const * payload, - ulong payload_sz ); +fd_failover_wire_encode( uchar * out, + ushort type, + uchar const * payload, + ulong payload_sz ); /* Decodes the frame at the start of buf and, on success, points *out_payload and *out_payload_sz into buf. */ int -fd_failover_wire_decode( fd_failover_wire_session_t * session, - uchar const * buf, - ulong buf_sz, - ushort * out_type, - uchar const ** out_payload, - ulong * out_payload_sz, - ulong * out_frame_sz ); +fd_failover_wire_decode( uchar const * buf, + ulong buf_sz, + ushort * out_type, + uchar const ** out_payload, + ulong * out_payload_sz, + ulong * out_frame_sz ); FD_PROTOTYPES_END diff --git a/src/discof/failover/test_failover_channel.c b/src/discof/failover/test_failover_channel.c index 4985d1dc7c6..ec11b20db03 100644 --- a/src/discof/failover/test_failover_channel.c +++ b/src/discof/failover/test_failover_channel.c @@ -27,7 +27,7 @@ member( ulong idx, ulong staked_idx ) { fd_failover_channel_t * ch = fd_failover_channel_join( fd_failover_channel_new( scratch[idx] ) ); FD_TEST( ch ); fd_failover_hello_t hello = { .version=FD_FAILOVER_VERSION, .boot_id=idx+1UL, - .role=(uchar)(idx ? FD_FAILOVER_ROLE_STANDBY : FD_FAILOVER_ROLE_ACTIVE), + .holds=(uchar)(idx ? 0 : 1), .mode=FD_FAILOVER_MODE_ALPENGLOW }; fd_memcpy( hello.junk_pubkey, keys[idx]+32UL, 32UL ); fd_memcpy( hello.staked_pubkey, keys[staked_idx]+32UL, 32UL ); diff --git a/src/discof/failover/test_failover_proto.c b/src/discof/failover/test_failover_proto.c index b72cdfa97d8..924a521e83d 100644 --- a/src/discof/failover/test_failover_proto.c +++ b/src/discof/failover/test_failover_proto.c @@ -13,25 +13,25 @@ fill_hello( fd_failover_hello_t * h, fd_memset( h->junk_pubkey, junk, 32UL ); fd_memset( h->staked_pubkey, staked, 32UL ); fd_memset( h->vote_account, vote, 32UL ); - h->role = role; + h->holds = role; h->mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; h->boot_id = 0x1000UL+junk; } static void test_hello( void ) { - fd_failover_hello_t active, standby; - fill_hello( &active, 1U, 0xAA, 0xBB, FD_FAILOVER_ROLE_ACTIVE ); - fill_hello( &standby, 2U, 0xAA, 0xBB, FD_FAILOVER_ROLE_STANDBY ); - FD_TEST( fd_failover_hello_check( &active, &standby )==FD_FAILOVER_HELLO_OK ); - FD_TEST( fd_failover_hello_check( &standby, &active )==FD_FAILOVER_HELLO_OK ); + fd_failover_hello_t holder, other; + fill_hello( &holder, 1U, 0xAA, 0xBB, 1 ); + fill_hello( &other, 2U, 0xAA, 0xBB, 0 ); + FD_TEST( fd_failover_hello_check( &holder, &other )==FD_FAILOVER_HELLO_OK ); + FD_TEST( fd_failover_hello_check( &other, &holder )==FD_FAILOVER_HELLO_OK ); uchar unsupported[] = { 0U, 2U, 255U }; for( ulong i=0UL; imode = FD_FAILOVER_MODE_ALPENGLOW; header->last_vote_slot = 102UL; - FD_TEST( !fd_failover_demoted_decode( &decoded, payload, sz ) ); + /* The wire layer does not read the history, the controller compares + the adopted tip with last_vote_slot. */ + FD_TEST( fd_failover_demoted_decode( &decoded, payload, sz ) && decoded.last_vote_slot==102UL ); } static void test_wrong_staked_key( void ) { - fd_failover_hello_t active, standby; - fill_hello( &active, 1U, 0xAA, 0xBB, FD_FAILOVER_ROLE_ACTIVE ); - fill_hello( &standby, 2U, 0xAC, 0xBB, FD_FAILOVER_ROLE_STANDBY ); - FD_TEST( fd_failover_hello_check( &active, &standby )==FD_FAILOVER_HELLO_ERR_STAKED ); + fd_failover_hello_t holder, other; + fill_hello( &holder, 1U, 0xAA, 0xBB, 1 ); + fill_hello( &other, 2U, 0xAC, 0xBB, 0 ); + FD_TEST( fd_failover_hello_check( &holder, &other )==FD_FAILOVER_HELLO_ERR_STAKED ); } /* test_handoff_decode: requests and results need their exact size and diff --git a/src/discof/failover/test_failover_tile.c b/src/discof/failover/test_failover_tile.c index 9eba738175f..95fbcd0fda3 100644 --- a/src/discof/failover/test_failover_tile.c +++ b/src/discof/failover/test_failover_tile.c @@ -38,16 +38,17 @@ static fd_failover_tile_ctx_t ctl[ 1 ]; #define OUR_BOOT_ID (1000UL) static fd_failover_tile_ctx_t * -controller_init( ulong role ) { +controller_init( int holds ) { stem_init(); fd_failover_tile_ctx_t * ctx = ctl; fd_memset( ctx, 0, sizeof(fd_failover_tile_ctx_t) ); - ctx->role = role; - ctx->hello.role = (uchar)role; + ctx->holds = holds; + ctx->hello.holds = (uchar)holds; ctx->hello.mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; ctx->hello.boot_id = OUR_BOOT_ID; fd_memset( ctx->hello.junk_pubkey, 0x11, 32UL ); fd_memset( ctx->hello.staked_pubkey, 0x5A, 32UL ); + fd_memcpy( ctx->installed, holds ? ctx->hello.staked_pubkey : ctx->hello.junk_pubkey, 32UL ); ctx->replay_slot = 100UL; ctx->root_slot = 90UL; ctx->last_vote_slot = 99UL; @@ -84,11 +85,11 @@ controller_fini( fd_failover_tile_ctx_t * ctx ) { static void pair( fd_failover_tile_ctx_t * ctx, ulong boot_id, - ulong peer_role, + ulong peer_holds, long now ) { ctx->channel->state = FD_FAILOVER_SESSION_PAIRED; ctx->channel->peer_hello.boot_id = boot_id; - ctx->channel->peer_hello.role = (uchar)peer_role; + ctx->channel->peer_hello.holds = (uchar)peer_holds; sync_session( ctx ); (void)now; fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x22, 32UL ); @@ -126,7 +127,7 @@ switch_ok( fd_failover_tile_ctx_t * ctx, ulong watermark ) { ctx->id_switch.result.result = FD_FAILOVER_SWITCH_OK; ctx->id_switch.result.history_watermark = watermark; - FD_TEST( switch_answer( ctx, ctx->id_switch.request_id ) ); + FD_TEST( on_switch_resp_msg( ctx, ctx->id_switch.request_id ) ); } static fd_failover_demoted_t @@ -139,18 +140,23 @@ pending_demoted( fd_failover_tile_ctx_t const * ctx ) { static void test_handoff( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); make_history( &ctx->current_history, 99UL ); fd_failover_handoff_request_t req = { .handoff_id=42UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH ); step_controller( ctx, stem ); FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK && !ctx->tx.valid ); switch_ok( ctx, 0UL ); step_controller( ctx, stem ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY ); + FD_TEST( ctx->holds==0 && fd_memeq( ctx->installed, ctx->hello.junk_pubkey, 32UL ) ); FD_TEST( pending_demoted( ctx ).handoff_id==42UL ); + fd_adminctl_failover_status_resp_t status; + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.version==FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION && !status.holds ); + FD_TEST( fd_memeq( status.target, ctx->hello.staked_pubkey, 32UL ) && fd_memeq( status.installed, ctx->hello.junk_pubkey, 32UL ) && + fd_memeq( status.junk, ctx->hello.junk_pubkey, 32UL ) ); ctx->tx.valid = 0; ctx->handoff.delivery = FD_FAILOVER_DELIVERY_SENT; deliver_ack( ctx, 42UL ); @@ -160,27 +166,62 @@ test_handoff( void ) { static void test_no_final_state( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); - fd_failover_handoff_request_t req = { .handoff_id=43UL, .target_boot_id=OUR_BOOT_ID }; + pair( ctx, 77UL, 0, 1000L ); + /* Pairing while we run the identity engages votor's failover rules. */ + FD_TEST( ctx->engage_pending ); + step_controller( ctx, stem ); + FD_TEST( !ctx->engage_pending && stem->seqs[ 0 ]==1UL && pub_mcache[ 0 ].sig==0UL && pub_mcache[ 0 ].sz==FD_VOTOR_ADOPT_ENGAGE_SZ ); + /* Before votor streams a full history a request is refused. */ + fd_failover_handoff_request_t req = { .handoff_id=43UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=100UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); fd_failover_handoff_result_t result; FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); FD_TEST( result.handoff_id==43UL && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_IDLE ); FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.request_id && !ctx->promotion.adopt_id ); + ctx->tx.valid = 0; + ctx->session.close_after_send = 0; + /* A frame still in plain mode has no history. */ + static fd_votor_hist_msg_t hist; + fd_memset( &hist, 0, sizeof(hist) ); + fd_memcpy( hist.identity.uc, ctx->hello.staked_pubkey, 32UL ); + hist.replay_slot = 100UL; + hist.root_slot = 90UL; + hist.vote_slot = FD_FAILOVER_SLOT_NULL; + hist.hist.last_leader_slot = FD_FAILOVER_SLOT_NULL; + hist.hist.vote_bound = FD_FAILOVER_SLOT_NULL; + on_votor_hist_msg( ctx, &hist ); + FD_TEST( ctx->last_vote_slot==FD_FAILOVER_SLOT_NULL && !ctx->current_history.valid ); + /* The first full frame after engagement comes from a completed slot, + not a vote, and still becomes our final history. */ + hist.hist.anchor = 99UL; + hist.hist.rec_cnt = 1UL; + hist.hist.rec[ 0 ].slot = 99UL; + hist.hist.rec[ 0 ].flags = AG_HIST_FLAG_VOTED; + hist.vote_slot = 99UL; + on_votor_hist_msg( ctx, &hist ); + FD_TEST( ctx->last_vote_slot==99UL && ctx->own_floor==99UL && ctx->current_history.valid && ctx->current_history.tip==99UL ); + req.handoff_id = 44UL; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( !ctx->tx.valid && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->handoff.id==44UL ); + /* A second pairing edge publishes the engage frame again. */ + ctx->channel->state = FD_FAILOVER_SESSION_BACKOFF; + sync_session( ctx ); + pair( ctx, 78UL, 0, 1000L ); + FD_TEST( ctx->engage_pending ); controller_fini( ctx ); } -/* test_request_replay_behind: a standby more than 64 slots behind our +/* test_request_replay_behind: a peer more than 64 slots behind our last vote, or one with no replay slot yet, gets REPLAY_BEHIND and we keep the identity, one exactly 64 behind is served. */ static void test_request_replay_behind( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); ctx->last_vote_slot = 200UL; make_history( &ctx->current_history, 200UL ); ulong behind[ 2 ] = { 200UL-FD_FAILOVER_DEADLINE_SLOTS-1UL, FD_FAILOVER_SLOT_NULL }; @@ -191,7 +232,7 @@ test_request_replay_behind( void ) { FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); FD_TEST( result.handoff_id==49UL+i && result.result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->handoff.id ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->handoff.id ); FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.request_id ); ctx->tx.valid = 0; ctx->session.close_after_send = 0; @@ -206,8 +247,8 @@ test_request_replay_behind( void ) { status reports it, and an ACK or the same request during it is ignored. */ static void test_demote_drain( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); make_history( &ctx->current_history, 99UL ); fd_failover_handoff_request_t req = { .handoff_id=44UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); @@ -215,7 +256,7 @@ test_demote_drain( void ) { ctx->hist_seen_seq = 796UL; switch_ok( ctx, 800UL ); step_controller( ctx, stem ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN ); fd_adminctl_failover_status_resp_t status; status_snapshot( ctx, 1000L, &status ); FD_TEST( status.action==(uchar)FD_FAILOVER_ACTION_DEMOTE_DRAIN ); @@ -224,7 +265,7 @@ test_demote_drain( void ) { FD_TEST( ctx->action==FD_FAILOVER_ACTION_DEMOTE_DRAIN && !ctx->taken && !ctx->tx.valid ); ctx->hist_seen_seq = 799UL; step_controller( ctx, stem ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && pending_demoted( ctx ).handoff_id==44UL ); + FD_TEST( ctx->holds==0 && pending_demoted( ctx ).handoff_id==44UL ); controller_fini( ctx ); } @@ -246,9 +287,9 @@ operator_frame( fd_failover_tile_ctx_t * ctx, fd_memset( operator.identity, key, 32UL ); fd_memset( operator.failover_identity, failover_key, 32UL ); fd_memcpy( msg->payload, &operator, sizeof(operator) ); - FD_TEST( !before_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR ) ); - during_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR, 0UL, sizeof(*msg), 0UL ); - after_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR, sizeof(*msg), 0UL, 0UL, stem ); + FD_TEST( !before_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR_ACTION ) ); + during_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR_ACTION, 0UL, sizeof(*msg), 0UL ); + after_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_OPERATOR_ACTION, sizeof(*msg), 0UL, 0UL, stem ); } /* test_operator_switch: set-identity during a demotion. We restart on @@ -257,8 +298,8 @@ operator_frame( fd_failover_tile_ctx_t * ctx, failover epoch, and a request bound to our old boot cannot demote us. */ static void test_operator_switch( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); make_history( &ctx->current_history, 99UL ); ctx->own_floor = 99UL; fd_failover_handoff_request_t req = { .handoff_id=47UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; @@ -269,7 +310,7 @@ test_operator_switch( void ) { /* set-identity installs the junk key. */ operator_frame( ctx, 1UL, 0x11, 0x5A ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && ctx->hello.role==FD_FAILOVER_ROLE_STANDBY ); + FD_TEST( ctx->holds==0 && ctx->hello.holds==0 && fd_memeq( ctx->installed, ctx->hello.junk_pubkey, 32UL ) ); FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->stuck && !ctx->tx.valid && !ctx->handoff.id && !ctx->request.id ); FD_TEST( ctx->hello.boot_id!=OUR_BOOT_ID && ctx->channel->self_hello.boot_id==ctx->hello.boot_id ); FD_TEST( !ctx->current_history.valid && ctx->own_floor==99UL ); @@ -278,13 +319,13 @@ test_operator_switch( void ) { /* The late answer ends the old switch and is dropped. */ ctx->id_switch.result.result = FD_FAILOVER_SWITCH_OK; - FD_TEST( !switch_answer( ctx, nonce ) ); + FD_TEST( !on_switch_resp_msg( ctx, nonce ) ); FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.fresh && !ctx->id_switch.discard ); /* The same or an older frame changes nothing. */ ulong boot_id = ctx->hello.boot_id; operator_frame( ctx, 1UL, 0x5A, 0x5A ); - FD_TEST( ctx->hello.boot_id==boot_id && ctx->role==FD_FAILOVER_ROLE_STANDBY ); + FD_TEST( ctx->hello.boot_id==boot_id && ctx->holds==0 ); /* The next switch request has the new failover epoch. */ FD_TEST( request_switch( ctx, stem, FD_FAILOVER_SWITCH_KEY_STAKED )!=ULONG_MAX ); @@ -301,12 +342,12 @@ test_operator_switch( void ) { fd_memset( ctx->id_switch.response.operator.failover_identity, 0x5A, 32UL ); ctx->id_switch.response_nonce = ctx->id_switch.request_id; after_frag( ctx, 1UL, 0UL, FD_FAILOVER_BUS_SWITCH_RESP, sizeof(fd_failover_bus_msg_t), 0UL, 0UL, stem ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->id_switch.epoch==2UL && ctx->hello.boot_id!=boot_id ); + FD_TEST( ctx->holds==1 && ctx->id_switch.epoch==2UL && ctx->hello.boot_id!=boot_id && fd_memeq( ctx->installed, ctx->hello.staked_pubkey, 32UL ) ); FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.fresh && ctx->action==FD_FAILOVER_ACTION_IDLE ); /* A request bound to our first boot is refused, one to this boot is served once we have a final history. */ - pair( ctx, 78UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + pair( ctx, 78UL, 0, 1000L ); deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE ); ctx->tx.valid = 0; @@ -320,32 +361,34 @@ test_operator_switch( void ) { } /* test_operator_failover_identity: HELLO has the failover identity - OPERATOR gives, and the role follows the installed key. */ + OPERATOR gives, and we run it when the installed key is that + identity. */ static void test_operator_failover_identity( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); ctx->staked_seen_at = 900L; operator_frame( ctx, 1UL, 0x77, 0x77 ); uchar third[ 32 ]; fd_memset( third, 0x77, 32UL ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->hello.role==FD_FAILOVER_ROLE_ACTIVE ); + FD_TEST( ctx->holds==1 && ctx->hello.holds==1 && fd_memeq( ctx->installed, third, 32UL ) ); FD_TEST( fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) && fd_memeq( ctx->channel->self_hello.staked_pubkey, third, 32UL ) ); FD_TEST( !ctx->staked_addr && !ctx->staked_seen_at ); /* A junk set-identity keeps the failover identity, the configured key does not come back. */ operator_frame( ctx, 2UL, 0x11, 0x77 ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) ); + FD_TEST( ctx->holds==0 && fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) ); - /* Any key but junk makes us the active, also one that is not the - failover identity in the frame. */ + /* Another installed key, as after failover promote prepared + the identity, means we do not run it. */ operator_frame( ctx, 3UL, 0x5A, 0x77 ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) ); + uchar other[ 32 ]; fd_memset( other, 0x5A, 32UL ); + FD_TEST( ctx->holds==0 && fd_memeq( ctx->hello.staked_pubkey, third, 32UL ) && fd_memeq( ctx->installed, other, 32UL ) ); /* A new target must be reflected in both admission and HELLO. */ operator_frame( ctx, 4UL, 0x11, 0x77 ); uchar public[ 32 ]; fd_memset( public, 0x5A, 32UL ); fd_failover_hello_t peer = ctx->channel->self_hello; - peer.role = FD_FAILOVER_ROLE_ACTIVE; + peer.holds = 1; peer.boot_id = 77UL; fd_memset( peer.junk_pubkey, 0x22, 32UL ); FD_TEST( fd_failover_hello_check( &ctx->channel->self_hello, &peer )==FD_FAILOVER_HELLO_OK ); @@ -359,7 +402,7 @@ test_operator_failover_identity( void ) { before we hear of it. */ static void test_standby_vote_counts( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); fd_votor_hist_msg_t hist; fd_memset( &hist, 0, sizeof(hist) ); fd_memcpy( hist.identity.uc, ctx->hello.staked_pubkey, 32UL ); @@ -367,13 +410,14 @@ test_standby_vote_counts( void ) { hist.root_slot = FD_FAILOVER_SLOT_NULL; hist.has_vote = 1; hist.vote_slot = 129UL; - consume_hist( ctx, &hist ); + on_votor_hist_msg( ctx, &hist ); FD_TEST( ctx->own_floor==129UL && ctx->last_vote_slot==129UL ); controller_fini( ctx ); } -/* A request of ours bound to boot 77 of the member whose junk key is - 0x22, as the tests' pair() makes it. */ +/* A request of ours bound to boot 77 of the member whose junk key was + 0x22, as the tests' pair() makes it. A new boot generates a new junk + key, so the restart tests pair with another one. */ static void open_request( fd_failover_tile_ctx_t * ctx, ulong action ) { @@ -388,33 +432,80 @@ open_request( fd_failover_tile_ctx_t * ctx, ctx->action = action; } -/* test_result_wait_peer_restart: the old active restarts before it - confirms our ACK. Its new boot cannot, so the wait ends and we stay - active. A standby waiting to have its refusal confirmed ends the - same way and stays standby. */ +/* test_result_wait_peer_restart: the peer that ran the identity + restarts before it confirms our ACK. Its new boot cannot, so the wait + ends and we keep the identity. A request waiting to have its refusal + confirmed ends the same way, without the identity. A request paused + at its deadline ends the same way when the new boot pairs, so the old + holder's next request is not refused as in progress forever. */ static void test_result_wait_peer_restart( void ) { - for( ulong active=0UL; active<2UL; active++ ) { - fd_failover_tile_ctx_t * ctx = controller_init( active ? FD_FAILOVER_ROLE_ACTIVE : FD_FAILOVER_ROLE_STANDBY ); - open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); - pair( ctx, 78UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); - int busy = 0; - peer_poll( ctx, 1000L, &busy ); - FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->request.id ); - FD_TEST( ctx->request.result==( active ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ) ); - FD_TEST( ctx->role==( active ? FD_FAILOVER_ROLE_ACTIVE : FD_FAILOVER_ROLE_STANDBY ) ); - FD_TEST( ctx->stuck==!active ); - controller_fini( ctx ); + for( int paused=0; paused<2; paused++ ) { + for( int holds=0; holds<2; holds++ ) { + fd_failover_tile_ctx_t * ctx = controller_init( holds ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + if( paused ) ctx->request.until = 0L; + pair( ctx, 78UL, 0, 1000L ); + fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x33, 32UL ); + int busy = 0; + peer_poll( ctx, stem, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->request.id && ctx->request.last_id==900UL ); + FD_TEST( ctx->request.result==( holds ? FD_FAILOVER_HANDOFF_TAKEN : FD_FAILOVER_HANDOFF_DECLINED ) ); + FD_TEST( ctx->holds==holds ); + FD_TEST( ctx->stuck==!holds ); + controller_fini( ctx ); + } } + /* A key switch in flight keeps the paused wait. */ + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + ctx->request.until = 0L; + ctx->id_switch.pending_key = FD_FAILOVER_SWITCH_KEY_STAKED; + pair( ctx, 78UL, 0, 1000L ); + fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x33, 32UL ); + int busy = 0; + peer_poll( ctx, stem, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && ctx->request.id==900UL && !ctx->request.until ); + controller_fini( ctx ); + + /* A paused request for final state keeps waiting for the operator. */ + ctx = controller_init( 0 ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ); + ctx->request.until = 0L; + pair( ctx, 78UL, 1, 1000L ); + fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x33, 32UL ); + busy = 0; + peer_poll( ctx, stem, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && ctx->request.id==900UL && !ctx->request.until && !ctx->request.peer_restarted ); + controller_fini( ctx ); + /* While we wait for final state a restart still pauses the request. */ - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + ctx = controller_init( 0 ); open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ); - pair( ctx, 78UL, FD_FAILOVER_ROLE_ACTIVE, 1000L ); - int busy = 0; - peer_poll( ctx, 1000L, &busy ); + pair( ctx, 78UL, 1, 1000L ); + fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x33, 32UL ); + busy = 0; + peer_poll( ctx, stem, 1000L, &busy ); FD_TEST( ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER && ctx->request.id && !ctx->request.until && ctx->stuck ); controller_fini( ctx ); + + /* `failover promote` that only ends a paused confirmation wait answers + with the id of the handoff it ended, not 0. */ + ctx = controller_init( 1 ); + open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + ctx->request.until = 0L; + ctx->request.peer_restarted = 1; + fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_HANDOFF }; + ctx->bus_req.nonce = 8UL; + fd_memcpy( ctx->bus_req.payload, &req, sizeof(req) ); + serve_bus_request( ctx, stem, 1000L ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->request.id && ctx->request.last_id==900UL && ctx->request.result==FD_FAILOVER_HANDOFF_TAKEN ); + fd_failover_bus_msg_t const * out = (fd_failover_bus_msg_t const *)bus_mem; + fd_adminctl_failover_control_resp_t response; + fd_memcpy( &response, out->payload, sizeof(response) ); + FD_TEST( out->nonce==8UL && out->result==FD_ADMINCTL_RESULT_SUCCESS && response.handoff_id==900UL && response.holds ); + controller_fini( ctx ); } /* test_session_change_drops_result: a queued HANDOFF_RESULT belongs to @@ -422,18 +513,18 @@ test_result_wait_peer_restart( void ) { queued DEMOTED stays for the handoff target. */ static void test_session_change_drops_result( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); fd_failover_handoff_request_t req = { .handoff_id=46UL, .target_boot_id=OUR_BOOT_ID+1UL, .replay_slot=99UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT && ctx->session.close_after_send ); ctx->channel->generation++; - pair( ctx, 79UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + pair( ctx, 79UL, 0, 1000L ); FD_TEST( !ctx->tx.valid && !ctx->session.close_after_send ); controller_fini( ctx ); - ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); make_history( &ctx->current_history, 99UL ); req.target_boot_id = OUR_BOOT_ID; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); @@ -442,7 +533,7 @@ test_session_change_drops_result( void ) { step_controller( ctx, stem ); FD_TEST( pending_demoted( ctx ).handoff_id==46UL ); ctx->channel->generation++; - pair( ctx, 80UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + pair( ctx, 80UL, 0, 1000L ); FD_TEST( pending_demoted( ctx ).handoff_id==46UL ); controller_fini( ctx ); } @@ -463,7 +554,7 @@ operator_cmd( fd_failover_tile_ctx_t * ctx, address too. */ static void test_request_address( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); ctx->staked_addr = FD_IP4_ADDR(10,0,0,3); @@ -504,16 +595,16 @@ test_request_address( void ) { fails keeps the identity and tells the requester at once. */ static void test_abort_answers_requester( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); make_history( &ctx->current_history, 99UL ); fd_failover_handoff_request_t req = { .handoff_id=47UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); step_controller( ctx, stem ); ctx->id_switch.result.result = FD_FAILOVER_SWITCH_ERR_DISABLED; - FD_TEST( switch_answer( ctx, ctx->id_switch.request_id ) ); + FD_TEST( on_switch_resp_msg( ctx, ctx->id_switch.request_id ) ); step_controller( ctx, stem ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE && ctx->stuck ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_IDLE && ctx->stuck ); fd_failover_handoff_result_t result; FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT && ctx->tx.to.boot_id==77UL && ctx->session.close_after_send ); FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); @@ -526,7 +617,7 @@ test_abort_answers_requester( void ) { and a restarted one ends the request. */ static void test_wait_result_redials( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); open_request( ctx, FD_FAILOVER_ACTION_PROMOTE_SWITCH ); ctx->request.until = 0L; ctx->promotion.from_peer = 1; @@ -536,49 +627,75 @@ test_wait_result_redials( void ) { ctx->id_switch.request_id = 1UL; switch_ok( ctx, 0UL ); step_controller( ctx, stem ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_HANDOFF_WAIT_RESULT && fd_memeq( ctx->installed, ctx->hello.staked_pubkey, 32UL ) ); FD_TEST( ctx->request.until && ctx->channel->peer_addr==FD_IP4_ADDR(10,0,0,2) ); controller_fini( ctx ); } /* test_refused_command_keeps_stuck: only an accepted command clears - stuck. */ + stuck, and the takeover checks behind a refusal report what they + find. */ static void test_refused_command_keeps_stuck( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); ctx->stuck = 1; - FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_FAILOVER_CONTROL_RESULT_NO_ACTIVE_ADDRESS && ctx->stuck ); + FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_FAILOVER_CONTROL_RESULT_NO_HOLDER_ADDRESS && ctx->stuck ); ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS && !ctx->stuck ); + request_end( ctx, 1000L, FD_FAILOVER_HANDOFF_DECLINED ); + controller_fini( ctx ); + + /* Without --force the takeover checks report SUCCESS when none finds + another machine running the identity, and the refusal otherwise. */ + ctx = controller_init( 0 ); + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + fd_adminctl_failover_status_resp_t status; + status_snapshot( ctx, 1000L, &status ); + FD_TEST( status.promote_result==FD_ADMINCTL_RESULT_SUCCESS && !status.holds ); + FD_TEST( fd_memeq( status.installed, ctx->hello.junk_pubkey, 32UL ) && fd_memeq( status.target, ctx->hello.staked_pubkey, 32UL ) ); + ctx->taken = 1; + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_FAILOVER_CONTROL_RESULT_TAKEN ); + FD_TEST( promote_guard( ctx, 1000L, 1 )==FD_ADMINCTL_RESULT_SUCCESS ); + ctx->taken = 0; + ctx->staked_seen_at = 1000L; + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_FAILOVER_CONTROL_RESULT_STAKED_SEEN ); + FD_TEST( promote_guard( ctx, 1000L+FD_FAILOVER_GOSSIP_FRESH_NANOS, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + ctx->staked_seen_at = 0L; + pair( ctx, 77UL, 1, 1000L ); + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_FAILOVER_CONTROL_RESULT_PEER_HOLDS ); + ctx->channel->peer_hello.holds = 0; + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); + ctx->holds = 1; + FD_TEST( promote_guard( ctx, 1000L, 0 )==FD_FAILOVER_CONTROL_RESULT_WRONG_HOLDER ); controller_fini( ctx ); } -/* test_dialed_standby: a request that pairs with a standby ends as its - own outcome, not as declined. */ +/* test_dialed_non_holder: a request that pairs with a peer that does not + run the identity ends as its own outcome, not as declined. */ static void -test_dialed_standby( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); +test_dialed_non_holder( void ) { + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); FD_TEST( operator_cmd( ctx, FD_ADMINCTL_FAILOVER_CMD_HANDOFF, 0U, 0 )==FD_ADMINCTL_RESULT_SUCCESS ); - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + pair( ctx, 77UL, 0, 1000L ); int busy = 0; - peer_poll( ctx, 1000L, &busy ); - FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->request.id && ctx->request.result==FD_FAILOVER_HANDOFF_NOT_ACTIVE ); + peer_poll( ctx, stem, 1000L, &busy ); + FD_TEST( ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->request.id && ctx->request.result==FD_FAILOVER_HANDOFF_NOT_HOLDER ); controller_fini( ctx ); } -/* test_restarted_peer_new_request: once the active boot a request is +/* test_restarted_peer_new_request: once the peer boot a request is bound to restarted, `failover promote` starts a new request instead of resuming one that can never finish, and the accepted command answers with its id. */ static void test_restarted_peer_new_request( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); ctx->staked_addr = FD_IP4_ADDR(10,0,0,2); open_request( ctx, FD_FAILOVER_ACTION_HANDOFF_WAIT_PEER ); - pair( ctx, 78UL, FD_FAILOVER_ROLE_ACTIVE, 1000L ); + pair( ctx, 78UL, 1, 1000L ); int busy = 0; - peer_poll( ctx, 1000L, &busy ); + peer_poll( ctx, stem, 1000L, &busy ); FD_TEST( ctx->request.id==900UL && ctx->request.peer_restarted && !ctx->request.until ); fd_adminctl_failover_req_t req = { .version=FD_ADMINCTL_FAILOVER_PAYLOAD_VERSION, .cmd=FD_ADMINCTL_FAILOVER_CMD_HANDOFF }; @@ -598,7 +715,7 @@ test_restarted_peer_new_request( void ) { backoff, whatever the last target reached. */ static void test_dial_backoff_reset( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); ctx->channel->backoff = ctx->channel->backoff_max; fd_failover_channel_init_dialer( ctx->channel, FD_IP4_ADDR(10,0,0,2), 8010 ); FD_TEST( ctx->channel->backoff==ctx->channel->backoff_min ); @@ -625,8 +742,8 @@ fill( fd_failover_channel_t * ch, static void test_listen_slot( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); - fd_failover_channel_t * ch = ctx->channel; + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + fd_failover_channel_t * ch = ctx->channel; long now = FD_FAILOVER_ADMISSION_START_NANOS; FD_TEST( listen_slot( ch, now )==UNAUTHENTICATED_IDX ); for( ulong i=0UL; irole = role; - ctx->hello.role = (uchar)role; + ctx->holds = holds; + ctx->hello.holds = (uchar)holds; ctx->hello.mode = (uchar)FD_FAILOVER_MODE_ALPENGLOW; ctx->hello.boot_id = OUR_BOOT_ID; fd_memset( ctx->hello.junk_pubkey, 0x11, 32UL ); fd_memset( ctx->hello.staked_pubkey, 0x5A, 32UL ); + fd_memcpy( ctx->installed, holds ? ctx->hello.staked_pubkey : ctx->hello.junk_pubkey, 32UL ); ctx->replay_slot = 100UL; ctx->root_slot = 90UL; ctx->last_vote_slot = 99UL; @@ -81,11 +82,11 @@ controller_fini( fd_failover_tile_ctx_t * ctx ) { static void pair( fd_failover_tile_ctx_t * ctx, ulong boot_id, - ulong peer_role, + ulong peer_holds, long now ) { ctx->channel->state = FD_FAILOVER_SESSION_PAIRED; ctx->channel->peer_hello.boot_id = boot_id; - ctx->channel->peer_hello.role = (uchar)peer_role; + ctx->channel->peer_hello.holds = (uchar)peer_holds; sync_session( ctx ); (void)now; fd_memset( ctx->channel->peer_hello.junk_pubkey, 0x22, 32UL ); @@ -110,7 +111,8 @@ make_hist( ag_hist_t * hist, } } -/* The same history serialized into history, as the active keeps it. */ +/* The same history serialized into history, as the machine running the + identity keeps it. */ static void make_history( fd_failover_history_t * history, ulong anchor, @@ -144,7 +146,7 @@ switch_ok( fd_failover_tile_ctx_t * ctx, ulong watermark ) { ctx->id_switch.result.result = FD_FAILOVER_SWITCH_OK; ctx->id_switch.result.history_watermark = watermark; - FD_TEST( switch_answer( ctx, ctx->id_switch.request_id ) ); + FD_TEST( on_switch_resp_msg( ctx, ctx->id_switch.request_id ) ); } static fd_failover_demoted_t @@ -155,21 +157,21 @@ pending_demoted( fd_failover_tile_ctx_t const * ctx ) { return demoted; } -/* test_handoff: the active hands over its vote history in an alpenglow - DEMOTED. */ +/* test_handoff: the machine running the identity hands over its vote + history in an alpenglow DEMOTED. */ static void test_handoff( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); make_history( &ctx->current_history, 90UL, 99UL, 6UL ); fd_failover_handoff_request_t req = { .handoff_id=42UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=99UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH ); step_controller( ctx, stem ); FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_JUNK && !ctx->tx.valid ); switch_ok( ctx, 0UL ); step_controller( ctx, stem ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY ); + FD_TEST( ctx->holds==0 ); fd_failover_demoted_t demoted = pending_demoted( ctx ); FD_TEST( demoted.handoff_id==42UL && demoted.mode==FD_FAILOVER_MODE_ALPENGLOW && demoted.last_vote_slot==99UL ); ctx->tx.valid = 0; @@ -179,25 +181,47 @@ test_handoff( void ) { controller_fini( ctx ); } -/* test_no_final_history: an active with no vote declines the handoff. */ +/* test_no_final_history: the machine running the identity declines the + handoff when it has no vote. */ static void test_no_final_history( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); ctx->last_vote_slot = FD_FAILOVER_SLOT_NULL; - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); - fd_failover_handoff_request_t req = { .handoff_id=43UL, .target_boot_id=OUR_BOOT_ID }; + pair( ctx, 77UL, 0, 1000L ); + /* Pairing while we run the identity engages votor's failover rules. */ + FD_TEST( ctx->engage_pending ); + step_controller( ctx, stem ); + FD_TEST( !ctx->engage_pending && stem->seqs[ 0 ]==1UL && pub_mcache[ 0 ].sig==0UL && pub_mcache[ 0 ].sz==FD_VOTOR_ADOPT_ENGAGE_SZ ); + /* Before votor streams a full history a request is refused. */ + fd_failover_handoff_request_t req = { .handoff_id=43UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=100UL }; deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); fd_failover_handoff_result_t result; FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); FD_TEST( result.handoff_id==43UL && result.result==FD_FAILOVER_CONTROL_RESULT_NO_FINAL_HISTORY ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_IDLE ); FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT && !ctx->id_switch.request_id && !ctx->promotion.adopt_id ); char const * hint; FD_TEST( !strcmp( control_refusal( result.result, &hint ), "NO_FINAL_HISTORY" ) ); - FD_TEST( strstr( hint, "the active" ) ); + FD_TEST( strstr( hint, "the other machine" ) ); /* The requester logs the relayed rejection under the same name. */ - FD_TEST( !strcmp( relayed_refusal( result.result, &hint ), "NO_FINAL_HISTORY" ) ); + FD_TEST( !strcmp( control_result( result.result, &hint ), "NO_FINAL_HISTORY" ) ); + ctx->tx.valid = 0; + ctx->session.close_after_send = 0; + /* The first full frame after engagement comes from a completed slot, + not a vote, and still becomes our final history. */ + static fd_votor_hist_msg_t hist; + fd_memset( &hist, 0, sizeof(hist) ); + fd_memcpy( hist.identity.uc, ctx->hello.staked_pubkey, 32UL ); + hist.replay_slot = 100UL; + hist.root_slot = 90UL; + make_hist( &hist.hist, 90UL, 96UL, 99UL, 6UL ); + hist.vote_slot = 99UL; + on_votor_hist_msg( ctx, &hist ); + FD_TEST( ctx->last_vote_slot==99UL && ctx->own_floor==99UL && ctx->current_history.valid && ctx->current_history.tip==99UL ); + req.handoff_id = 44UL; + deliver( ctx, FD_FAILOVER_MSG_HANDOFF_REQUEST, &req, sizeof(req) ); + FD_TEST( !ctx->tx.valid && ctx->action==FD_FAILOVER_ACTION_DEMOTE_SWITCH && ctx->handoff.id==44UL ); controller_fini( ctx ); } @@ -205,8 +229,8 @@ test_no_final_history( void ) { replay is within the slot deadline of the history's finality anchor. */ static void test_request_replay_behind( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_ACTIVE ); - pair( ctx, 77UL, FD_FAILOVER_ROLE_STANDBY, 1000L ); + fd_failover_tile_ctx_t * ctx = controller_init( 1 ); + pair( ctx, 77UL, 0, 1000L ); ctx->last_vote_slot = 300UL; make_history( &ctx->current_history, 200UL, 300UL, 6UL ); fd_failover_handoff_request_t req = { .handoff_id=52UL, .target_boot_id=OUR_BOOT_ID, .replay_slot=200UL-FD_FAILOVER_DEADLINE_SLOTS-1UL }; @@ -215,7 +239,7 @@ test_request_replay_behind( void ) { FD_TEST( ctx->tx.valid && ctx->tx.type==FD_FAILOVER_MSG_HANDOFF_RESULT ); FD_TEST( fd_failover_handoff_result_decode( &result, ctx->tx.payload, ctx->tx.sz ) ); FD_TEST( result.handoff_id==52UL && result.result==FD_FAILOVER_CONTROL_RESULT_REPLAY_BEHIND ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_ACTIVE && ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->handoff.id ); + FD_TEST( ctx->holds==1 && ctx->action==FD_FAILOVER_ACTION_IDLE && !ctx->handoff.id ); ctx->tx.valid = 0; ctx->session.close_after_send = 0; req.handoff_id = 53UL; @@ -230,7 +254,7 @@ test_request_replay_behind( void ) { higher. */ static void test_force_empty_bound( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); fd_cstr_ncpy( ctx->promotion.label, "promotion 1", sizeof(ctx->promotion.label) ); ctx->promotion.source = FD_FAILOVER_SOURCE_EMPTY; ctx->promotion.force = 1; @@ -248,7 +272,7 @@ test_force_empty_bound( void ) { static void test_adopt_capacity_retry( void ) { - fd_failover_tile_ctx_t * ctx = controller_init( FD_FAILOVER_ROLE_STANDBY ); + fd_failover_tile_ctx_t * ctx = controller_init( 0 ); make_history( &ctx->peer_history, 90UL, 99UL, 6UL ); fd_failover_peer_t peer = { .boot_id = 77UL }; start_promotion( ctx, FD_FAILOVER_SOURCE_PEER, &peer, 42UL, 0 ); @@ -260,7 +284,7 @@ test_adopt_capacity_retry( void ) { after_frag( ctx, 0UL, 0UL, id, sizeof(ctx->adopt_result), 0UL, 0UL, stem ); step_promote_wait_adopt( ctx, stem ); FD_TEST( ctx->promotion.retry && ctx->action==FD_FAILOVER_ACTION_PROMOTE_WAIT_ADOPT ); - FD_TEST( ctx->role==FD_FAILOVER_ROLE_STANDBY && !ctx->stuck && !ctx->tx.valid ); + FD_TEST( ctx->holds==0 && !ctx->stuck && !ctx->tx.valid ); FD_TEST( ctx->id_switch.pending_key==FD_FAILOVER_SWITCH_KEY_CNT ); FD_TEST( ctx->peer_history.valid && fd_memeq( &saved, &ctx->promotion.adopt, sizeof(saved) ) ); step_promote_wait_adopt( ctx, stem ); @@ -269,7 +293,7 @@ test_adopt_capacity_retry( void ) { ctx->replay_slot++; step_promote_wait_adopt( ctx, stem ); FD_TEST( !ctx->promotion.retry && ctx->promotion.adopt_id!=id ); - FD_TEST( fd_memeq( bus_mem, saved.state, saved.sz ) ); + FD_TEST( fd_memeq( bus_mem, ctx->hello.staked_pubkey, FD_VOTOR_ADOPT_REQ_HDR_SZ ) && fd_memeq( bus_mem+FD_VOTOR_ADOPT_REQ_HDR_SZ, saved.state, saved.sz ) ); ctx->adopt_result = (fd_votor_adopt_result_t){ .result=FD_VOTOR_ADOPT_SUCCESS, .root=90UL, .vote_slot=99UL, .vote_bound=ULONG_MAX }; after_frag( ctx, 0UL, 1UL, ctx->promotion.adopt_id, sizeof(ctx->adopt_result), 0UL, 0UL, stem ); step_promote_wait_adopt( ctx, stem ); diff --git a/src/discof/failover/test_failover_wire.c b/src/discof/failover/test_failover_wire.c index 0f5d557276c..9a1e1eff256 100644 --- a/src/discof/failover/test_failover_wire.c +++ b/src/discof/failover/test_failover_wire.c @@ -4,37 +4,28 @@ static uchar frame[ FD_FAILOVER_FRAME_MAX ]; static uchar big[ 2UL*FD_FAILOVER_FRAME_MAX ]; -static void -paired_sessions( fd_failover_wire_session_t * a, - fd_failover_wire_session_t * b ) { - fd_failover_wire_session_init( a ); - fd_failover_wire_session_init( b ); -} - static void test_roundtrip( void ) { - fd_failover_wire_session_t a, b; - paired_sessions( &a, &b ); uchar payload[ 70 ]; for( ulong i=0UL; i<70UL; i++ ) payload[ i ] = (uchar)i; - ulong sz = fd_failover_wire_encode( &a, frame, (ushort)FD_FAILOVER_MSG_DEMOTED, payload, 70UL ); + ulong sz = fd_failover_wire_encode( frame, (ushort)FD_FAILOVER_MSG_DEMOTED, payload, 70UL ); FD_TEST( sz==FD_FAILOVER_FRAME_HDR_SZ+70UL ); ushort type; uchar const * out; ulong out_sz; ulong frame_sz; - FD_TEST( fd_failover_wire_decode( &b, frame, sz, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); + FD_TEST( fd_failover_wire_decode( frame, sz, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); FD_TEST( type==(ushort)FD_FAILOVER_MSG_DEMOTED ); FD_TEST( out_sz==70UL ); FD_TEST( frame_sz==sz ); FD_TEST( fd_memeq( out, payload, 70UL ) ); - ulong sz2 = fd_failover_wire_encode( &a, big, (ushort)FD_FAILOVER_MSG_HELLO, payload, 10UL ); + ulong sz2 = fd_failover_wire_encode( big, (ushort)FD_FAILOVER_MSG_HELLO, payload, 10UL ); fd_memset( big+sz2, 0x77, 64UL ); - FD_TEST( fd_failover_wire_decode( &b, big, sz2+64UL, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); + FD_TEST( fd_failover_wire_decode( big, sz2+64UL, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); FD_TEST( frame_sz==sz2 ); FD_LOG_NOTICE(( "pass: test_roundtrip" )); @@ -42,19 +33,18 @@ test_roundtrip( void ) { static void test_replay( void ) { - fd_failover_wire_session_t a, b; - paired_sessions( &a, &b ); uchar payload[ 27 ] = { 5 }; - ulong sz = fd_failover_wire_encode( &a, frame, (ushort)FD_FAILOVER_MSG_DEMOTED, payload, 27UL ); + ulong sz = fd_failover_wire_encode( frame, (ushort)FD_FAILOVER_MSG_DEMOTED, payload, 27UL ); ushort type; uchar const * out; ulong out_sz; ulong frame_sz; - FD_TEST( fd_failover_wire_decode( &b, frame, sz, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); - FD_TEST( fd_failover_wire_decode( &b, frame, sz, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_ERR_SEQ ); + FD_TEST( fd_failover_wire_decode( frame, sz, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); + /* Frames carry no sequence, TLS orders and protects the stream. */ + FD_TEST( fd_failover_wire_decode( frame, sz, &type, &out, &out_sz, &frame_sz )==FD_FAILOVER_WIRE_SUCCESS ); FD_LOG_NOTICE(( "pass: test_replay" )); } diff --git a/src/discof/gossip/fd_gossvf_tile.c b/src/discof/gossip/fd_gossvf_tile.c index db311cc291d..adc141eaf61 100644 --- a/src/discof/gossip/fd_gossvf_tile.c +++ b/src/discof/gossip/fd_gossvf_tile.c @@ -627,8 +627,8 @@ check_duplicate_instance( fd_gossvf_tile_ctx_t * ctx, if( FD_UNLIKELY( their_creation_nanos>our_creation_nanos && !memcmp( ctx->identity_pubkey->uc, value->origin, 32UL ) ) ) { FD_LOG_ERR(( "duplicate running instances of the same validator node, our timestamp: %ldns their timestamp: %ldns. " - "If the newer instance is a failover peer booted without `[failover.enabled]`, stop it and enable failover " - "there, then run `firedancer failover promote --force` on the machine that should hold the identity.", + "If the newer instance is a failover machine that restarted with this identity in [paths.identity_key], stop it, " + "set its [paths.identity_key] to a random junk key and restart it, then check `failover status` on both machines.", our_creation_nanos, their_creation_nanos )); } } diff --git a/src/discof/votor/fd_votor_tile.c b/src/discof/votor/fd_votor_tile.c index 37614ef8acc..1d0f219113a 100644 --- a/src/discof/votor/fd_votor_tile.c +++ b/src/discof/votor/fd_votor_tile.c @@ -396,14 +396,16 @@ struct fd_votor_tile { /* Failover, on when the failover links are in the topology */ int failover_enabled; - int failover_engaged; /* failover voting rules apply, from boot on a standby, from the first adoption on a client */ - fd_pubkey_t boot_id_key; /* the identity we booted with, the junk key under failover */ + int failover_engaged; /* failover voting rules apply, from the first adoption or the pairing while we run the identity */ + fd_pubkey_t junk_key; /* the junk key generated at boot, a handoff switches to it */ + ulong highest_unsent_slot; /* highest vote the tile dropped before failover engaged, ULONG_MAX none */ int failover_hist_adopted; /* a history, possibly empty, was adopted since the last switch */ int vote_authority; /* our rank may be installed, 0 keeps us unranked */ int hist_was_truncated; /* the last export dropped windows, warned once */ ulong last_leader_slot; /* slot of the last LEADER we published, ULONG_MAX before any */ ulong adopted_last_leader_slot; /* window of the adopted last LEADER or vote bound, ULONG_MAX when none */ ulong last_vote_slot; /* highest slot we broadcast a vote for, ULONG_MAX before any */ + fd_pubkey_t last_vote_key; /* the identity that vote went out as */ ulong root_slot; /* replay's root at the last completed slot, ULONG_MAX before any */ ulong highest_completed_slot; /* highest slot replay completed, 0 before any */ @@ -420,7 +422,7 @@ struct fd_votor_tile { ulong failov_out_wmark; ulong failov_out_chunk; - uchar adopt_req[ AG_HIST_SER_MAX ]; + uchar adopt_req[ FD_VOTOR_ADOPT_REQ_HDR_SZ+AG_HIST_SER_MAX ]; ulong adopt_req_sz; }; typedef struct fd_votor_tile fd_votor_tile_t; @@ -1416,19 +1418,20 @@ raise_leader_floor( fd_votor_tile_t * ctx, if( FD_LIKELY( ctx->adopted_last_leader_slot==ULONG_MAX || ctx->adopted_last_leader_slotadopted_last_leader_slot = window; } -/* An empty request is shorter than any serialized history. */ -FD_STATIC_ASSERT( FD_VOTOR_ADOPT_EMPTY_SZsizeof(ctx->adopt_req) || ag_hist_de( req, req_sz, hist ) ) ) ) { + if( FD_UNLIKELY( !empty && ( req_sz>AG_HIST_SER_MAX || ag_hist_de( req, req_sz, hist ) ) ) ) { FD_LOG_WARNING(( "failover: refusing a vote history of %lu bytes that does not decode", req_sz )); return result; } @@ -1464,9 +1467,10 @@ failover_adopt_hist( fd_votor_tile_t * ctx, return result; } - /* Reject a history older than the votes we sent as the staked identity. */ + /* Reject a history older than the votes we sent as THIS identity. */ result.result = FD_VOTOR_ADOPT_ERR_STALE; - if( FD_UNLIKELY( ctx->failover_engaged && ctx->last_vote_slot!=ULONG_MAX && tiplast_vote_slot ) ) { + if( FD_UNLIKELY( ctx->failover_engaged && ctx->last_vote_slot!=ULONG_MAX && tiplast_vote_slot && + fd_pubkey_eq( target, &ctx->last_vote_key ) ) ) { FD_LOG_WARNING(( "failover: refusing a vote history through slot %lu, this machine already voted as this identity through slot %lu", tip, ctx->last_vote_slot )); return result; } @@ -1510,14 +1514,14 @@ failover_switched( fd_votor_tile_t * ctx ) { ctx->next_leader_slot = fd_multi_epoch_leaders_get_next_slot( ctx->mleaders, fd_ulong_max( fd_ulong_max( ctx->curr_epoch_slot, floor_window ), ctx->vote_history_slot ), &ctx->id_key ); } } - int operator = !!FD_VOLATILE_CONST( ctx->id_keyswitch->operator ); - int standby = fd_pubkey_eq( &ctx->id_key, &ctx->boot_id_key ); - ctx->vote_authority = ( !ctx->failover_engaged && operator ) || ( !standby && ( operator || ctx->failover_hist_adopted ) ); + int operator = !FD_VOLATILE_CONST( ctx->id_keyswitch->failover ); + int junk = fd_pubkey_eq( &ctx->id_key, &ctx->junk_key ); + ctx->vote_authority = operator || ( !junk && ctx->failover_hist_adopted ); if( FD_UNLIKELY( ctx->failover_engaged ) ) { FD_BASE58_ENCODE_32_BYTES( ctx->id_key.uc, id_key_b58 ); - FD_LOG_NOTICE(( "failover: switched to identity `%s`%s, this machine is now %s", id_key_b58, operator ? " by set-identity" : "", - ctx->vote_authority ? "the active voter" : "a standby" )); - if( FD_UNLIKELY( !operator && !standby && !ctx->failover_hist_adopted ) ) FD_LOG_WARNING(( "failover: the staked identity was installed without an adopted vote history, refusing to vote until an adoption precedes an identity switch" )); + FD_LOG_NOTICE(( "failover: switched to identity `%s`%s, this machine %s", id_key_b58, operator ? " by set-identity" : "", + ctx->vote_authority ? "votes as it" : "does not vote as it" )); + if( FD_UNLIKELY( !operator && !junk && !ctx->failover_hist_adopted ) ) FD_LOG_WARNING(( "failover: the identity was installed without an adopted vote history, refusing to vote until an adoption precedes an identity switch" )); } ctx->failover_hist_adopted = 0; ctx->id_keyswitch->result = ctx->hist_seq; @@ -1540,8 +1544,15 @@ failover_after_vote( fd_votor_tile_t * ctx, fd_stem_context_t * stem, ulong vote_slot, int sent ) { - if( FD_LIKELY( sent && ( ctx->last_vote_slot==ULONG_MAX || vote_slot>ctx->last_vote_slot ) ) ) ctx->last_vote_slot = vote_slot; - if( FD_LIKELY( !ctx->failover_engaged ) ) return; + if( FD_LIKELY( sent && ( ctx->last_vote_slot==ULONG_MAX || vote_slot>ctx->last_vote_slot ) ) ) { + ctx->last_vote_slot = vote_slot; + ctx->last_vote_key = ctx->id_key; + } + if( FD_LIKELY( !ctx->failover_engaged ) ) { + /* Raise highest_unsent_slot if needed. */ + if( FD_UNLIKELY( !sent && ( ctx->highest_unsent_slot==ULONG_MAX || vote_slot>ctx->highest_unsent_slot ) ) ) ctx->highest_unsent_slot = vote_slot; + return; + } if( FD_UNLIKELY( !sent ) ) ag_votor_mark_unsent( ctx->votor, &ctx->scratch.vote ); publish_hist( ctx, stem, sent ); } @@ -1556,6 +1567,36 @@ failover_after_leader( fd_votor_tile_t * ctx, publish_hist( ctx, stem, 0 ); } +/* The highest slot our slot states may record a vote at that did not go + out under three special cases: 1.) the restart floor, 2.) the catch-up floor, + 3.) and the highest vote the tile dropped before failover engaged. + ULONG_MAX is returned when there is no such special bound. */ +static ulong +failover_engage_bound( fd_votor_tile_t const * ctx ) { + ulong slots[ 3 ] = { ctx->vote_history_slot ? ctx->vote_history_slot-1UL : ULONG_MAX, + ctx->catchup_vote_slot ? ctx->catchup_vote_slot-1UL : ULONG_MAX, + ctx->highest_unsent_slot }; + ulong bound = ULONG_MAX; + for( ulong i=0UL; i<3UL; i++ ) { + if( slots[ i ]==ULONG_MAX ) continue; + ulong end = ag_first_slot_in_window( slots[ i ] )+AG_SLOTS_PER_WINDOW-1UL; + bound = bound==ULONG_MAX ? end : fd_ulong_max( bound, end ); + } + return bound; +} + +/* The failover tile paired with the other machine while we run the + identity, so the failover voting rules engage, as a latch. */ +static void +failover_engage( fd_votor_tile_t * ctx ) { + if( FD_LIKELY( ctx->failover_engaged ) ) return; + ulong bound = failover_engage_bound( ctx ); + if( FD_LIKELY( bound!=ULONG_MAX ) ) ag_votor_set_vote_bound( ctx->votor, bound ); + ctx->failover_engaged = 1; + FD_BASE58_ENCODE_32_BYTES( ctx->id_key.uc, id_key_b58 ); + FD_LOG_NOTICE(( "failover: paired with the other machine while running identity `%s`, the failover voting rules engage", id_key_b58 )); +} + /* failover_init initializes failover in the votor tile, all of it off unless the topology has the failover links. */ static void @@ -1583,11 +1624,13 @@ failover_init( fd_votor_tile_t * ctx, ctx->failov_out_chunk = ctx->failov_out_chunk0; } - /* A configured standby waits for a switch. An outgoing client keeps - ordinary voting until its first successful adoption. */ - ctx->boot_id_key = ctx->id_key; - ctx->failover_engaged = !!tile->votor.failover_standby; - ctx->vote_authority = !tile->votor.failover_standby; + /* Every validator votes as its boot identity with the usual voting rules + until failover engages, at its first adoption or when the failover + tile pairs while this machine runs the identity. */ + fd_memcpy( ctx->junk_key.uc, topo->failover_junk, 32UL ); + ctx->failover_engaged = 0; + ctx->vote_authority = 1; + ctx->highest_unsent_slot = ULONG_MAX; ctx->failover_hist_adopted = 0; ctx->hist_was_truncated = 0; ctx->last_leader_slot = ULONG_MAX; @@ -1612,25 +1655,28 @@ failover_adopt_copy( fd_votor_tile_t * ctx, /* Responds to an adoption request, echoing its sequence number. */ static void -failover_adopt_answer( fd_votor_tile_t * ctx, +failover_adopt_response( fd_votor_tile_t * ctx, ulong sig, fd_stem_context_t * stem ) { + if( FD_UNLIKELY( ctx->adopt_req_sz==FD_VOTOR_ADOPT_ENGAGE_SZ ) ) { + failover_engage( ctx ); + return; + } fd_votor_adopt_result_t result; - if( FD_UNLIKELY( !fd_pubkey_eq( &ctx->id_key, &ctx->boot_id_key ) || ( ctx->vote_authority && ctx->failover_engaged ) ) ) { - FD_LOG_WARNING(( "failover: refusing a vote history adoption outside standby or client startup" )); + if( FD_UNLIKELY( ctx->adopt_req_szadopt_req_sz>sizeof(ctx->adopt_req) ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history adoption of %lu bytes, it has no target identity", ctx->adopt_req_sz )); + result = (fd_votor_adopt_result_t){ .result=FD_VOTOR_ADOPT_ERR_DECODE, .root=ULONG_MAX, .vote_slot=ULONG_MAX, .vote_bound=ULONG_MAX }; + } else if( FD_UNLIKELY( ctx->vote_authority && ctx->failover_engaged ) ) { + FD_LOG_WARNING(( "failover: refusing a vote history adoption, this machine votes as an identity failover handed it" )); result = (fd_votor_adopt_result_t){ .result=FD_VOTOR_ADOPT_ERR_INVALID, .root=ULONG_MAX, .vote_slot=ULONG_MAX, .vote_bound=ULONG_MAX }; } else { - /* Our slot states hold the votes of the boot identity, a standby's - junk key or a client's own key, and none of them went out as the - target. Fence their whole windows so the target cannot finalize - them or build on them. Export the bound with future histories so - another machine preserves it. */ ulong boot_bound = ULONG_MAX; ag_hist_t local; ag_votor_hist_export( ctx->votor, ULONG_MAX, &local ); ulong tip = ag_hist_tip( &local ); if( tip!=ULONG_MAX ) boot_bound = ag_first_slot_in_window( tip )+AG_SLOTS_PER_WINDOW-1UL; - result = failover_adopt_hist( ctx, ctx->adopt_req, ctx->adopt_req_sz ); + result = failover_adopt_hist( ctx, (fd_pubkey_t const *)ctx->adopt_req, ctx->adopt_req+FD_VOTOR_ADOPT_REQ_HDR_SZ, + ctx->adopt_req_sz-FD_VOTOR_ADOPT_REQ_HDR_SZ ); if( FD_LIKELY( result.result==FD_VOTOR_ADOPT_SUCCESS ) ) { if( boot_bound!=ULONG_MAX ) { ag_votor_set_vote_bound( ctx->votor, boot_bound ); @@ -2806,7 +2852,7 @@ after_frag( fd_votor_tile_t * ctx, if( FD_UNLIKELY( ctx->failover_enabled && sig==REPLAY_SIG_SLOT_COMPLETED ) ) publish_hist( ctx, stem, 0 ); /* keeps the failover deadlines ticking */ break; case IN_KIND_FAILOV: - failover_adopt_answer( ctx, sig, stem ); + failover_adopt_response( ctx, sig, stem ); break; default: FD_LOG_ERR(( "unexpected in_kind %d", ctx->in_kind[ in_idx ] )); @@ -2856,20 +2902,16 @@ privileged_init( fd_topo_t const * topo, if( FD_UNLIKELY( -1==faccessat( ctx->vote_history_fd_dir, bin, F_OK, 0 ) ) ) { if( FD_UNLIKELY( errno!=ENOENT ) ) FD_LOG_ERR(( "faccessat(`%s/%s`) failed (%i-%s)", tile->votor.vote_history_path, bin, errno, fd_io_strerror( errno ) )); - /* A generated passive identity has never voted. Keep existing - voting identities' files, including their backups, untouched. */ - if( !tile->votor.failover_standby || strcmp( tile->votor.identity_key_path, FD_KEYLOAD_FAILOVER_JUNK ) ) { - DIR * dir = opendir( tile->votor.vote_history_path ); - if( FD_UNLIKELY( !dir ) ) FD_LOG_ERR(( "opendir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); - errno = 0; - for( struct dirent * ent; (ent = readdir( dir )); ) { - if( FD_LIKELY( strncmp( ent->d_name, "vote_history-", sizeof("vote_history-")-1UL ) ) ) continue; - FD_LOG_ERR(( "Booting as %s, but detected %s in %s, which is missing %s. Please either copy %s into %s or remove the vote_history-* files there if you wish to boot without a vote history file.", - id_key_b58, ent->d_name, tile->votor.vote_history_path, bin, bin, tile->votor.vote_history_path )); - } - if( FD_UNLIKELY( errno ) ) FD_LOG_ERR(( "readdir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); - if( FD_UNLIKELY( closedir( dir ) ) ) FD_LOG_ERR(( "closedir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); + DIR * dir = opendir( tile->votor.vote_history_path ); + if( FD_UNLIKELY( !dir ) ) FD_LOG_ERR(( "opendir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); + errno = 0; + for( struct dirent * ent; (ent = readdir( dir )); ) { + if( FD_LIKELY( strncmp( ent->d_name, "vote_history-", sizeof("vote_history-")-1UL ) ) ) continue; + FD_LOG_ERR(( "Booting as %s, but detected %s in %s, which is missing %s. Please either copy %s into %s or remove the vote_history-* files there if you wish to boot without a vote history file.", + id_key_b58, ent->d_name, tile->votor.vote_history_path, bin, bin, tile->votor.vote_history_path )); } + if( FD_UNLIKELY( errno ) ) FD_LOG_ERR(( "readdir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); + if( FD_UNLIKELY( closedir( dir ) ) ) FD_LOG_ERR(( "closedir(`%s`) failed (%i-%s)", tile->votor.vote_history_path, errno, fd_io_strerror( errno ) )); FD_LOG_WARNING(( "vote history file `%s/%s` does not exist, creating it. This identity is assumed to have never voted.", tile->votor.vote_history_path, bin )); } ctx->vote_history_slot = read_vote_history( ctx, tile->votor.vote_history_path, bin ); diff --git a/src/discof/votor/fd_votor_tile.h b/src/discof/votor/fd_votor_tile.h index 1704f9e5437..27171ec138b 100644 --- a/src/discof/votor/fd_votor_tile.h +++ b/src/discof/votor/fd_votor_tile.h @@ -145,7 +145,13 @@ typedef struct fd_votor_hist_msg fd_votor_hist_msg_t; /* Adoption of a peer's vote history. */ -#define FD_VOTOR_ADOPT_EMPTY_SZ (8UL) /* an empty request, a slot as a ulong */ +/* Frame sizes */ + +#define FD_VOTOR_ADOPT_REQ_HDR_SZ (32UL) /* the identity the adoption is for, before the payload */ +#define FD_VOTOR_ADOPT_EMPTY_SZ (8UL) /* an empty payload, a slot as a ulong */ +#define FD_VOTOR_ADOPT_ENGAGE_SZ (0UL) /* no header or payload, engages failover voting rules */ + +/* Result codes */ #define FD_VOTOR_ADOPT_SUCCESS (0UL) #define FD_VOTOR_ADOPT_ERR_DECODE (1UL) diff --git a/src/discof/votor/test_votor_tile.c b/src/discof/votor/test_votor_tile.c index a81b2de467a..b5c2cf8cd2d 100644 --- a/src/discof/votor/test_votor_tile.c +++ b/src/discof/votor/test_votor_tile.c @@ -1753,7 +1753,7 @@ hist_setup( fd_votor_tile_t * ctx ) { build_stakes( stakes, 3UL, 10UL ); ag_epoch_info_t * epoch_info = rank_voters( &epoch_info_mem, stakes, 3UL, NULL, NULL ); memcpy( ctx->id_key.uc, epoch_info->validators[0].id_key, sizeof(fd_pubkey_t) ); - ctx->boot_id_key = ctx->id_key; + ctx->junk_key = ctx->id_key; ctx->curr_epoch_info = epoch_info; ctx->failover_enabled = 1; ctx->failover_engaged = 1; @@ -2670,7 +2670,7 @@ test_history_raises_the_leader_floor( void ) { uchar req[ AG_HIST_SER_MAX ]; ulong req_sz; FD_TEST( !ag_hist_ser( &hist, req, sizeof(req), &req_sz ) ); - fd_votor_adopt_result_t result = failover_adopt_hist( &ctx, req, req_sz ); + fd_votor_adopt_result_t result = failover_adopt_hist( &ctx, &ctx.id_key, req, req_sz ); FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS && result.vote_slot==1UL && result.vote_bound==17UL ); FD_TEST( ag_votor_vote_bound( ctx.votor )==17UL ); FD_TEST( ctx.adopted_last_leader_slot==ag_first_slot_in_window( 17UL ) ); @@ -2679,20 +2679,25 @@ test_history_raises_the_leader_floor( void ) { hist.last_leader_slot = 4UL; hist.vote_bound = 6UL; FD_TEST( !ag_hist_ser( &hist, req, sizeof(req), &req_sz ) ); - result = failover_adopt_hist( &ctx, req, req_sz ); + result = failover_adopt_hist( &ctx, &ctx.id_key, req, req_sz ); FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS && result.vote_bound==17UL ); FD_TEST( ctx.adopted_last_leader_slot==ag_first_slot_in_window( 17UL ) ); uchar empty[ FD_VOTOR_ADOPT_EMPTY_SZ ]; FD_STORE( ulong, empty, 2UL ); - result = failover_adopt_hist( &ctx, empty, sizeof(empty) ); + result = failover_adopt_hist( &ctx, &ctx.id_key, empty, sizeof(empty) ); FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS && result.vote_bound==17UL ); FD_TEST( ctx.adopted_last_leader_slot==ag_first_slot_in_window( 17UL ) ); - /* A history older than the votes this machine sent is rejected. */ + /* A history older than the votes this machine sent as that identity + is rejected, one for another identity is not. */ ctx.last_vote_slot = 3UL; - result = failover_adopt_hist( &ctx, req, req_sz ); + ctx.last_vote_key = ctx.id_key; + result = failover_adopt_hist( &ctx, &ctx.id_key, req, req_sz ); FD_TEST( result.result==FD_VOTOR_ADOPT_ERR_STALE ); + fd_pubkey_t other; fd_memset( other.uc, 0x77, 32UL ); + result = failover_adopt_hist( &ctx, &other, req, req_sz ); + FD_TEST( result.result==FD_VOTOR_ADOPT_SUCCESS ); hist_teardown( &ctx ); } @@ -2807,8 +2812,9 @@ test_failover_off( void ) { hist_teardown( &ctx ); } -/* Test that with failover the staked key votes only after an adoption or - set-identity, and the junk key never does. */ +/* Test that with failover a failover identity switch starts voting only + after adoption, and that set-identity votes as on main, even to the junk + key. */ static void operator_switch( int operator, @@ -2821,7 +2827,7 @@ operator_switch( int operator, init_keys( &ctx, auth_vtr_mem, &bls_key, 1UL ); /* the new identity's key, as the resume loads it */ ctx.id_keyswitch = fd_keyswitch_join( fd_keyswitch_new( id_keyswitch_mem, FD_KEYSWITCH_STATE_SWITCH_PENDING ) ); FD_TEST( ctx.id_keyswitch ); - ctx.id_keyswitch->operator = (ulong)operator; + ctx.id_keyswitch->failover = (ulong)!operator; ctx.hist_seq = 77UL; memcpy( ctx.id_key.uc, epoch_info->validators[ new_rank ].id_key, sizeof(fd_pubkey_t) ); @@ -2870,7 +2876,8 @@ test_switch_runs_the_failover_rule( void ) { fd_clock_tile_init( ctx.clock ); /* A failover switch to the junk key we booted under, no adoption. */ - memcpy( ctx.id_keyswitch->bytes, ctx.boot_id_key.uc, sizeof(fd_pubkey_t) ); + memcpy( ctx.id_keyswitch->bytes, ctx.junk_key.uc, sizeof(fd_pubkey_t) ); + ctx.id_keyswitch->failover = 1UL; fd_keyswitch_state( ctx.id_keyswitch, FD_KEYSWITCH_STATE_SWITCH_PENDING ); during_housekeeping( &ctx ); FD_TEST( ctx.id_keyswitch->state==FD_KEYSWITCH_STATE_COMPLETED ); @@ -2883,7 +2890,7 @@ static void test_operator_switch_votes( void ) { operator_switch( 0, 1UL, 0 ); /* staked, no adoption */ operator_switch( 1, 1UL, 1 ); /* staked, by set-identity */ - operator_switch( 1, 0UL, 0 ); /* back to the junk key, by set-identity */ + operator_switch( 1, 0UL, 1 ); /* to the junk key, by set-identity */ } /* Test that an adoption is rejected while this machine votes and taken @@ -2912,10 +2919,11 @@ test_adopt_refused_while_voting( void ) { ctx.failov_out_wmark = 4UL; ctx.failov_out_chunk = 0UL; ctx.in_kind[ 0 ] = IN_KIND_FAILOV; - FD_STORE( ulong, ctx.adopt_req, 9UL ); /* empty history, no vote at or below slot 9 */ - ctx.adopt_req_sz = FD_VOTOR_ADOPT_EMPTY_SZ; + fd_memcpy( ctx.adopt_req, ctx.id_key.uc, FD_VOTOR_ADOPT_REQ_HDR_SZ ); + FD_STORE( ulong, ctx.adopt_req+FD_VOTOR_ADOPT_REQ_HDR_SZ, 9UL ); /* empty history, no vote at or below slot 9 */ + ctx.adopt_req_sz = FD_VOTOR_ADOPT_REQ_HDR_SZ+FD_VOTOR_ADOPT_EMPTY_SZ; - after_frag( &ctx, 0UL, 0UL, 901UL, FD_VOTOR_ADOPT_EMPTY_SZ, 0UL, 0UL, stem ); + after_frag( &ctx, 0UL, 0UL, 901UL, ctx.adopt_req_sz, 0UL, 0UL, stem ); fd_votor_adopt_result_t const * reply = fd_chunk_to_laddr_const( reply_mem, mcache[0].chunk ); FD_TEST( seqs[0]==1UL && mcache[0].sig==901UL ); FD_TEST( reply->result==FD_VOTOR_ADOPT_ERR_INVALID && !ctx.failover_hist_adopted ); @@ -2928,7 +2936,8 @@ test_adopt_refused_while_voting( void ) { hist.rec[ i ].slot = 32UL+i; hist.rec[ i ].flags = AG_HIST_FLAG_VOTED; } - FD_TEST( !ag_hist_ser( &hist, ctx.adopt_req, sizeof(ctx.adopt_req), &ctx.adopt_req_sz ) ); + FD_TEST( !ag_hist_ser( &hist, ctx.adopt_req+FD_VOTOR_ADOPT_REQ_HDR_SZ, AG_HIST_SER_MAX, &ctx.adopt_req_sz ) ); + ctx.adopt_req_sz += FD_VOTOR_ADOPT_REQ_HDR_SZ; ulong free = ag_votor_metrics( ctx.votor ).slot_state_pool_free; after_frag( &ctx, 0UL, 1UL, 902UL, ctx.adopt_req_sz, 0UL, 0UL, stem ); reply = fd_chunk_to_laddr_const( reply_mem, mcache[1].chunk ); @@ -2942,14 +2951,26 @@ test_adopt_refused_while_voting( void ) { ctx.failover_engaged = 1; ctx.vote_authority = 0; - FD_STORE( ulong, ctx.adopt_req, 9UL ); - ctx.adopt_req_sz = FD_VOTOR_ADOPT_EMPTY_SZ; - after_frag( &ctx, 0UL, 2UL, 903UL, FD_VOTOR_ADOPT_EMPTY_SZ, 0UL, 0UL, stem ); + FD_STORE( ulong, ctx.adopt_req+FD_VOTOR_ADOPT_REQ_HDR_SZ, 9UL ); + ctx.adopt_req_sz = FD_VOTOR_ADOPT_REQ_HDR_SZ+FD_VOTOR_ADOPT_EMPTY_SZ; + after_frag( &ctx, 0UL, 2UL, 903UL, ctx.adopt_req_sz, 0UL, 0UL, stem ); reply = fd_chunk_to_laddr_const( reply_mem, mcache[2].chunk ); FD_TEST( seqs[0]==3UL && mcache[2].sig==903UL ); FD_TEST( reply->result==FD_VOTOR_ADOPT_SUCCESS && ctx.failover_hist_adopted ); FD_TEST( ag_votor_vote_bound( ctx.votor )==9UL ); + /* An engage frame (empty) engages the latch for using the failover votor rules idempotently. */ + ctx.failover_engaged = 0; + ctx.highest_unsent_slot = 21UL; /* its window ends at 23 */ + ctx.adopt_req_sz = FD_VOTOR_ADOPT_ENGAGE_SZ; + after_frag( &ctx, 0UL, 3UL, 0UL, ctx.adopt_req_sz, 0UL, 0UL, stem ); + FD_TEST( seqs[0]==3UL && ctx.failover_engaged && !ctx.vote_authority ); + FD_TEST( ag_votor_vote_bound( ctx.votor )==23UL ); + ctx.highest_unsent_slot = 40UL; + after_frag( &ctx, 0UL, 4UL, 0UL, ctx.adopt_req_sz, 0UL, 0UL, stem ); + FD_TEST( seqs[0]==3UL && ctx.failover_engaged ); + FD_TEST( ag_votor_vote_bound( ctx.votor )==23UL ); + hist_teardown( &ctx ); } From f5b27f6483ef7cf9782c74fa7bb9377dd4431cb3 Mon Sep 17 00:00:00 2001 From: Charles Moyes Date: Wed, 28 Oct 2026 01:22:12 +0000 Subject: [PATCH 23/23] votor: persist the adopted signing constraints at a failover switch A promotion adopts the other machine's vote history, vote bound and leader floor into consensus state, which a restart loses. The admin tile passes a zero floor for a failover switch, so the vote history file written at unhalt held only the local watermark. If the adopted tip or bound was ahead of it, a crash before the next local vote or leader window booted without those constraints and could sign in conflict with the votes the other machine cast as the identity. Record the adopted tip, and when a failover switch grants vote authority raise the watermark to cover it, the vote bound and the leader floor, so the file this identity boots from waits past them. set-identity is unchanged, its floor comes from the operator's file. Co-Authored-By: Claude Fable 5.1 --- src/discof/votor/fd_votor_tile.c | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/src/discof/votor/fd_votor_tile.c b/src/discof/votor/fd_votor_tile.c index 1d0f219113a..bf22c4eb131 100644 --- a/src/discof/votor/fd_votor_tile.c +++ b/src/discof/votor/fd_votor_tile.c @@ -400,6 +400,7 @@ struct fd_votor_tile { fd_pubkey_t junk_key; /* the junk key generated at boot, a handoff switches to it */ ulong highest_unsent_slot; /* highest vote the tile dropped before failover engaged, ULONG_MAX none */ int failover_hist_adopted; /* a history, possibly empty, was adopted since the last switch */ + ulong adopted_tip; /* tip of the adopted history, ULONG_MAX for an empty one or none */ int vote_authority; /* our rank may be installed, 0 keeps us unranked */ int hist_was_truncated; /* the last export dropped windows, warned once */ ulong last_leader_slot; /* slot of the last LEADER we published, ULONG_MAX before any */ @@ -1454,6 +1455,7 @@ failover_adopt_hist( fd_votor_tile_t * ctx, ag_votor_set_vote_bound( ctx->votor, bound ); raise_leader_floor( ctx, bound ); ctx->failover_hist_adopted = 1; + ctx->adopted_tip = ULONG_MAX; result.result = FD_VOTOR_ADOPT_SUCCESS; result.root = ag_votor_metrics( ctx->votor ).highest_final_cert_slot; result.vote_bound = ag_votor_vote_bound( ctx->votor ); @@ -1490,6 +1492,7 @@ failover_adopt_hist( fd_votor_tile_t * ctx, } if( FD_UNLIKELY( conflicts ) ) FD_LOG_WARNING(( "adopted vote history disagrees with %lu of our own notar hashes, theirs were the ones sent", conflicts )); ctx->failover_hist_adopted = 1; + ctx->adopted_tip = tip; raise_leader_floor( ctx, hist->last_leader_slot ); raise_leader_floor( ctx, ag_votor_vote_bound( ctx->votor ) ); result.result = FD_VOTOR_ADOPT_SUCCESS; @@ -1517,6 +1520,20 @@ failover_switched( fd_votor_tile_t * ctx ) { int operator = !FD_VOLATILE_CONST( ctx->id_keyswitch->failover ); int junk = fd_pubkey_eq( &ctx->id_key, &ctx->junk_key ); ctx->vote_authority = operator || ( !junk && ctx->failover_hist_adopted ); + + /* A promotion votes past the adopted history's tip, its vote bound + and the leader floor, which are consensus state and lost on a + restart. The vote history file written at unhalt holds only the + watermark, so raise it to cover them and the file this identity + boots from after a crash waits past them too. The old identity's + file was written at the halt and is not affected. */ + + if( FD_UNLIKELY( ctx->vote_authority && !operator ) ) { + ulong slots[ 3 ] = { ctx->adopted_tip, ag_votor_vote_bound( ctx->votor ), floor }; + for( ulong i=0UL; i<3UL; i++ ) { + if( FD_LIKELY( slots[ i ]!=ULONG_MAX ) ) ctx->highest_vote_or_lead_slot = fd_ulong_max( ctx->highest_vote_or_lead_slot, slots[ i ] ); + } + } if( FD_UNLIKELY( ctx->failover_engaged ) ) { FD_BASE58_ENCODE_32_BYTES( ctx->id_key.uc, id_key_b58 ); FD_LOG_NOTICE(( "failover: switched to identity `%s`%s, this machine %s", id_key_b58, operator ? " by set-identity" : "", @@ -1524,6 +1541,7 @@ failover_switched( fd_votor_tile_t * ctx ) { if( FD_UNLIKELY( !operator && !junk && !ctx->failover_hist_adopted ) ) FD_LOG_WARNING(( "failover: the identity was installed without an adopted vote history, refusing to vote until an adoption precedes an identity switch" )); } ctx->failover_hist_adopted = 0; + ctx->adopted_tip = ULONG_MAX; ctx->id_keyswitch->result = ctx->hist_seq; } @@ -1632,6 +1650,7 @@ failover_init( fd_votor_tile_t * ctx, ctx->vote_authority = 1; ctx->highest_unsent_slot = ULONG_MAX; ctx->failover_hist_adopted = 0; + ctx->adopted_tip = ULONG_MAX; ctx->hist_was_truncated = 0; ctx->last_leader_slot = ULONG_MAX; ctx->adopted_last_leader_slot = ULONG_MAX;